1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the AArch64 target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_imm0_31(int64_t Imm) {
12
13 return ((uint64_t)Imm) < 32;
14
15}
16static bool Predicate_imm0_63(int64_t Imm) {
17
18 return ((uint64_t)Imm) < 64;
19
20}
21static bool Predicate_imm32_0_31(int64_t Imm) {
22
23 return ((uint64_t)Imm) < 32;
24
25}
26static bool Predicate_tbz_imm0_31_diag(int64_t Imm) {
27
28 return (((uint32_t)Imm) < 32);
29
30}
31static bool Predicate_tbz_imm32_63(int64_t Imm) {
32
33 return (((uint32_t)Imm) > 31) && (((uint32_t)Imm) < 64);
34
35}
36static bool Predicate_VectorIndexD(int64_t Imm) {
37 return ((uint64_t)Imm) < 2;
38}
39static bool Predicate_VectorIndexS(int64_t Imm) {
40 return ((uint64_t)Imm) < 4;
41}
42static bool Predicate_VectorIndexH(int64_t Imm) {
43 return ((uint64_t)Imm) < 8;
44}
45static bool Predicate_VectorIndexB(int64_t Imm) {
46 return ((uint64_t)Imm) < 16;
47}
48static bool Predicate_VectorIndex0(int64_t Imm) {
49 return ((uint64_t)Imm) == 0;
50}
51static bool Predicate_imm0_255(int64_t Imm) {
52
53 return ((uint32_t)Imm) < 256;
54
55}
56static bool Predicate_simm8_32b(int64_t Imm) {
57 return Imm >= -128 && Imm < 128;
58}
59static bool Predicate_simm8_64b(int64_t Imm) {
60 return Imm >= -128 && Imm < 128;
61}
62static bool Predicate_uimm8_32b(int64_t Imm) {
63 return Imm >= 0 && Imm < 256;
64}
65static bool Predicate_uimm8_64b(int64_t Imm) {
66 return Imm >= 0 && Imm < 256;
67}
68static bool Predicate_simm6_32b(int64_t Imm) {
69 return Imm >= -32 && Imm < 32;
70}
71
72
73// FastEmit functions for AArch64ISD::ENTRY_PSTATE_SM.
74
75Register fastEmit_AArch64ISD_ENTRY_PSTATE_SM_MVT_i64_(MVT RetVT) {
76 if (RetVT.SimpleTy != MVT::i64)
77 return Register();
78 return fastEmitInst_(MachineInstOpcode: AArch64::EntryPStateSM, RC: &AArch64::GPR64RegClass);
79}
80
81Register fastEmit_AArch64ISD_ENTRY_PSTATE_SM_(MVT VT, MVT RetVT) {
82 switch (VT.SimpleTy) {
83 case MVT::i64: return fastEmit_AArch64ISD_ENTRY_PSTATE_SM_MVT_i64_(RetVT);
84 default: return Register();
85 }
86}
87
88// FastEmit functions for AArch64ISD::GET_SME_SAVE_SIZE.
89
90Register fastEmit_AArch64ISD_GET_SME_SAVE_SIZE_MVT_i64_(MVT RetVT) {
91 if (RetVT.SimpleTy != MVT::i64)
92 return Register();
93 return fastEmitInst_(MachineInstOpcode: AArch64::GetSMESaveSize, RC: &AArch64::GPR64RegClass);
94}
95
96Register fastEmit_AArch64ISD_GET_SME_SAVE_SIZE_(MVT VT, MVT RetVT) {
97 switch (VT.SimpleTy) {
98 case MVT::i64: return fastEmit_AArch64ISD_GET_SME_SAVE_SIZE_MVT_i64_(RetVT);
99 default: return Register();
100 }
101}
102
103// FastEmit functions for AArch64ISD::THREAD_POINTER.
104
105Register fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(MVT RetVT) {
106 if (RetVT.SimpleTy != MVT::i64)
107 return Register();
108 return fastEmitInst_(MachineInstOpcode: AArch64::MOVbaseTLS, RC: &AArch64::GPR64RegClass);
109}
110
111Register fastEmit_AArch64ISD_THREAD_POINTER_(MVT VT, MVT RetVT) {
112 switch (VT.SimpleTy) {
113 case MVT::i64: return fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(RetVT);
114 default: return Register();
115 }
116}
117
118// Top-level FastEmit function.
119
120Register fastEmit_(MVT VT, MVT RetVT, unsigned Opcode) override {
121 switch (Opcode) {
122 case AArch64ISD::ENTRY_PSTATE_SM: return fastEmit_AArch64ISD_ENTRY_PSTATE_SM_(VT, RetVT);
123 case AArch64ISD::GET_SME_SAVE_SIZE: return fastEmit_AArch64ISD_GET_SME_SAVE_SIZE_(VT, RetVT);
124 case AArch64ISD::THREAD_POINTER: return fastEmit_AArch64ISD_THREAD_POINTER_(VT, RetVT);
125 default: return Register();
126 }
127}
128
129// FastEmit functions for AArch64ISD::ALLOCATE_ZA_BUFFER.
130
131Register fastEmit_AArch64ISD_ALLOCATE_ZA_BUFFER_MVT_i64_r(MVT RetVT, Register Op0) {
132 if (RetVT.SimpleTy != MVT::i64)
133 return Register();
134 return fastEmitInst_r(MachineInstOpcode: AArch64::AllocateZABuffer, RC: &AArch64::GPR64spRegClass, Op0);
135}
136
137Register fastEmit_AArch64ISD_ALLOCATE_ZA_BUFFER_r(MVT VT, MVT RetVT, Register Op0) {
138 switch (VT.SimpleTy) {
139 case MVT::i64: return fastEmit_AArch64ISD_ALLOCATE_ZA_BUFFER_MVT_i64_r(RetVT, Op0);
140 default: return Register();
141 }
142}
143
144// FastEmit functions for AArch64ISD::ALLOC_SME_SAVE_BUFFER.
145
146Register fastEmit_AArch64ISD_ALLOC_SME_SAVE_BUFFER_MVT_i64_r(MVT RetVT, Register Op0) {
147 if (RetVT.SimpleTy != MVT::i64)
148 return Register();
149 return fastEmitInst_r(MachineInstOpcode: AArch64::AllocateSMESaveBuffer, RC: &AArch64::GPR64spRegClass, Op0);
150}
151
152Register fastEmit_AArch64ISD_ALLOC_SME_SAVE_BUFFER_r(MVT VT, MVT RetVT, Register Op0) {
153 switch (VT.SimpleTy) {
154 case MVT::i64: return fastEmit_AArch64ISD_ALLOC_SME_SAVE_BUFFER_MVT_i64_r(RetVT, Op0);
155 default: return Register();
156 }
157}
158
159// FastEmit functions for AArch64ISD::CALL.
160
161Register fastEmit_AArch64ISD_CALL_MVT_i64_r(MVT RetVT, Register Op0) {
162 if (RetVT.SimpleTy != MVT::isVoid)
163 return Register();
164 if (( MF->getSubtarget<AArch64Subtarget>().hardenSlsBlr() )) {
165 return fastEmitInst_r(MachineInstOpcode: AArch64::BLRNoIP, RC: &AArch64::GPR64noipRegClass, Op0);
166 }
167 if (( !MF->getSubtarget<AArch64Subtarget>().hardenSlsBlr() )) {
168 return fastEmitInst_r(MachineInstOpcode: AArch64::BLR, RC: &AArch64::GPR64RegClass, Op0);
169 }
170 return Register();
171}
172
173Register fastEmit_AArch64ISD_CALL_r(MVT VT, MVT RetVT, Register Op0) {
174 switch (VT.SimpleTy) {
175 case MVT::i64: return fastEmit_AArch64ISD_CALL_MVT_i64_r(RetVT, Op0);
176 default: return Register();
177 }
178}
179
180// FastEmit functions for AArch64ISD::COALESCER_BARRIER.
181
182Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_bf16_r(MVT RetVT, Register Op0) {
183 if (RetVT.SimpleTy != MVT::bf16)
184 return Register();
185 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR16, RC: &AArch64::FPR16RegClass, Op0);
186}
187
188Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f16_r(MVT RetVT, Register Op0) {
189 if (RetVT.SimpleTy != MVT::f16)
190 return Register();
191 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR16, RC: &AArch64::FPR16RegClass, Op0);
192}
193
194Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f32_r(MVT RetVT, Register Op0) {
195 if (RetVT.SimpleTy != MVT::f32)
196 return Register();
197 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR32, RC: &AArch64::FPR32RegClass, Op0);
198}
199
200Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f64_r(MVT RetVT, Register Op0) {
201 if (RetVT.SimpleTy != MVT::f64)
202 return Register();
203 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
204}
205
206Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f128_r(MVT RetVT, Register Op0) {
207 if (RetVT.SimpleTy != MVT::f128)
208 return Register();
209 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
210}
211
212Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i8_r(MVT RetVT, Register Op0) {
213 if (RetVT.SimpleTy != MVT::v8i8)
214 return Register();
215 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
216}
217
218Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v16i8_r(MVT RetVT, Register Op0) {
219 if (RetVT.SimpleTy != MVT::v16i8)
220 return Register();
221 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
222}
223
224Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i16_r(MVT RetVT, Register Op0) {
225 if (RetVT.SimpleTy != MVT::v4i16)
226 return Register();
227 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
228}
229
230Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i16_r(MVT RetVT, Register Op0) {
231 if (RetVT.SimpleTy != MVT::v8i16)
232 return Register();
233 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
234}
235
236Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i32_r(MVT RetVT, Register Op0) {
237 if (RetVT.SimpleTy != MVT::v2i32)
238 return Register();
239 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
240}
241
242Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i32_r(MVT RetVT, Register Op0) {
243 if (RetVT.SimpleTy != MVT::v4i32)
244 return Register();
245 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
246}
247
248Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1i64_r(MVT RetVT, Register Op0) {
249 if (RetVT.SimpleTy != MVT::v1i64)
250 return Register();
251 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
252}
253
254Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i64_r(MVT RetVT, Register Op0) {
255 if (RetVT.SimpleTy != MVT::v2i64)
256 return Register();
257 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
258}
259
260Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f16_r(MVT RetVT, Register Op0) {
261 if (RetVT.SimpleTy != MVT::v4f16)
262 return Register();
263 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
264}
265
266Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8f16_r(MVT RetVT, Register Op0) {
267 if (RetVT.SimpleTy != MVT::v8f16)
268 return Register();
269 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
270}
271
272Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4bf16_r(MVT RetVT, Register Op0) {
273 if (RetVT.SimpleTy != MVT::v4bf16)
274 return Register();
275 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
276}
277
278Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8bf16_r(MVT RetVT, Register Op0) {
279 if (RetVT.SimpleTy != MVT::v8bf16)
280 return Register();
281 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
282}
283
284Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f32_r(MVT RetVT, Register Op0) {
285 if (RetVT.SimpleTy != MVT::v2f32)
286 return Register();
287 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
288}
289
290Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f32_r(MVT RetVT, Register Op0) {
291 if (RetVT.SimpleTy != MVT::v4f32)
292 return Register();
293 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
294}
295
296Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1f64_r(MVT RetVT, Register Op0) {
297 if (RetVT.SimpleTy != MVT::v1f64)
298 return Register();
299 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
300}
301
302Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f64_r(MVT RetVT, Register Op0) {
303 if (RetVT.SimpleTy != MVT::v2f64)
304 return Register();
305 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
306}
307
308Register fastEmit_AArch64ISD_COALESCER_BARRIER_r(MVT VT, MVT RetVT, Register Op0) {
309 switch (VT.SimpleTy) {
310 case MVT::bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_bf16_r(RetVT, Op0);
311 case MVT::f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f16_r(RetVT, Op0);
312 case MVT::f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f32_r(RetVT, Op0);
313 case MVT::f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f64_r(RetVT, Op0);
314 case MVT::f128: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f128_r(RetVT, Op0);
315 case MVT::v8i8: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i8_r(RetVT, Op0);
316 case MVT::v16i8: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v16i8_r(RetVT, Op0);
317 case MVT::v4i16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i16_r(RetVT, Op0);
318 case MVT::v8i16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i16_r(RetVT, Op0);
319 case MVT::v2i32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i32_r(RetVT, Op0);
320 case MVT::v4i32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i32_r(RetVT, Op0);
321 case MVT::v1i64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1i64_r(RetVT, Op0);
322 case MVT::v2i64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i64_r(RetVT, Op0);
323 case MVT::v4f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f16_r(RetVT, Op0);
324 case MVT::v8f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8f16_r(RetVT, Op0);
325 case MVT::v4bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4bf16_r(RetVT, Op0);
326 case MVT::v8bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8bf16_r(RetVT, Op0);
327 case MVT::v2f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f32_r(RetVT, Op0);
328 case MVT::v4f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f32_r(RetVT, Op0);
329 case MVT::v1f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1f64_r(RetVT, Op0);
330 case MVT::v2f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f64_r(RetVT, Op0);
331 default: return Register();
332 }
333}
334
335// FastEmit functions for AArch64ISD::DUP.
336
337Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Register Op0) {
338 if ((Subtarget->isNeonAvailable())) {
339 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv8i8gpr, RC: &AArch64::FPR64RegClass, Op0);
340 }
341 return Register();
342}
343
344Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Register Op0) {
345 if ((Subtarget->isNeonAvailable())) {
346 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv16i8gpr, RC: &AArch64::FPR128RegClass, Op0);
347 }
348 return Register();
349}
350
351Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Register Op0) {
352 if ((Subtarget->isNeonAvailable())) {
353 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv4i16gpr, RC: &AArch64::FPR64RegClass, Op0);
354 }
355 return Register();
356}
357
358Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Register Op0) {
359 if ((Subtarget->isNeonAvailable())) {
360 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv8i16gpr, RC: &AArch64::FPR128RegClass, Op0);
361 }
362 return Register();
363}
364
365Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Register Op0) {
366 if ((Subtarget->isNeonAvailable())) {
367 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv2i32gpr, RC: &AArch64::FPR64RegClass, Op0);
368 }
369 return Register();
370}
371
372Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Register Op0) {
373 if ((Subtarget->isNeonAvailable())) {
374 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv4i32gpr, RC: &AArch64::FPR128RegClass, Op0);
375 }
376 return Register();
377}
378
379Register fastEmit_AArch64ISD_DUP_MVT_i32_r(MVT RetVT, Register Op0) {
380switch (RetVT.SimpleTy) {
381 case MVT::v8i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Op0);
382 case MVT::v16i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Op0);
383 case MVT::v4i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Op0);
384 case MVT::v8i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Op0);
385 case MVT::v2i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Op0);
386 case MVT::v4i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Op0);
387 default: return Register();
388}
389}
390
391Register fastEmit_AArch64ISD_DUP_MVT_i64_r(MVT RetVT, Register Op0) {
392 if (RetVT.SimpleTy != MVT::v2i64)
393 return Register();
394 if ((Subtarget->isNeonAvailable())) {
395 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv2i64gpr, RC: &AArch64::FPR128RegClass, Op0);
396 }
397 return Register();
398}
399
400Register fastEmit_AArch64ISD_DUP_r(MVT VT, MVT RetVT, Register Op0) {
401 switch (VT.SimpleTy) {
402 case MVT::i32: return fastEmit_AArch64ISD_DUP_MVT_i32_r(RetVT, Op0);
403 case MVT::i64: return fastEmit_AArch64ISD_DUP_MVT_i64_r(RetVT, Op0);
404 default: return Register();
405 }
406}
407
408// FastEmit functions for AArch64ISD::FCVTXN.
409
410Register fastEmit_AArch64ISD_FCVTXN_MVT_f64_r(MVT RetVT, Register Op0) {
411 if (RetVT.SimpleTy != MVT::f32)
412 return Register();
413 if ((Subtarget->isNeonAvailable())) {
414 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTXNv1i64, RC: &AArch64::FPR32RegClass, Op0);
415 }
416 return Register();
417}
418
419Register fastEmit_AArch64ISD_FCVTXN_MVT_v2f64_r(MVT RetVT, Register Op0) {
420 if (RetVT.SimpleTy != MVT::v2f32)
421 return Register();
422 if ((Subtarget->isNeonAvailable())) {
423 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTXNv2f32, RC: &AArch64::FPR64RegClass, Op0);
424 }
425 return Register();
426}
427
428Register fastEmit_AArch64ISD_FCVTXN_r(MVT VT, MVT RetVT, Register Op0) {
429 switch (VT.SimpleTy) {
430 case MVT::f64: return fastEmit_AArch64ISD_FCVTXN_MVT_f64_r(RetVT, Op0);
431 case MVT::v2f64: return fastEmit_AArch64ISD_FCVTXN_MVT_v2f64_r(RetVT, Op0);
432 default: return Register();
433 }
434}
435
436// FastEmit functions for AArch64ISD::FRECPE.
437
438Register fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(MVT RetVT, Register Op0) {
439 if (RetVT.SimpleTy != MVT::v2f32)
440 return Register();
441 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv2f32, RC: &AArch64::FPR64RegClass, Op0);
442}
443
444Register fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(MVT RetVT, Register Op0) {
445 if (RetVT.SimpleTy != MVT::v4f32)
446 return Register();
447 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv4f32, RC: &AArch64::FPR128RegClass, Op0);
448}
449
450Register fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(MVT RetVT, Register Op0) {
451 if (RetVT.SimpleTy != MVT::v2f64)
452 return Register();
453 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv2f64, RC: &AArch64::FPR128RegClass, Op0);
454}
455
456Register fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
457 if (RetVT.SimpleTy != MVT::nxv8f16)
458 return Register();
459 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
460 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
461 }
462 return Register();
463}
464
465Register fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
466 if (RetVT.SimpleTy != MVT::nxv4f32)
467 return Register();
468 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
469 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
470 }
471 return Register();
472}
473
474Register fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
475 if (RetVT.SimpleTy != MVT::nxv2f64)
476 return Register();
477 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
478 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
479 }
480 return Register();
481}
482
483Register fastEmit_AArch64ISD_FRECPE_r(MVT VT, MVT RetVT, Register Op0) {
484 switch (VT.SimpleTy) {
485 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(RetVT, Op0);
486 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(RetVT, Op0);
487 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(RetVT, Op0);
488 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(RetVT, Op0);
489 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(RetVT, Op0);
490 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(RetVT, Op0);
491 default: return Register();
492 }
493}
494
495// FastEmit functions for AArch64ISD::FRSQRTE.
496
497Register fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(MVT RetVT, Register Op0) {
498 if (RetVT.SimpleTy != MVT::v2f32)
499 return Register();
500 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv2f32, RC: &AArch64::FPR64RegClass, Op0);
501}
502
503Register fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(MVT RetVT, Register Op0) {
504 if (RetVT.SimpleTy != MVT::v4f32)
505 return Register();
506 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv4f32, RC: &AArch64::FPR128RegClass, Op0);
507}
508
509Register fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(MVT RetVT, Register Op0) {
510 if (RetVT.SimpleTy != MVT::v2f64)
511 return Register();
512 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv2f64, RC: &AArch64::FPR128RegClass, Op0);
513}
514
515Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
516 if (RetVT.SimpleTy != MVT::nxv8f16)
517 return Register();
518 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
519 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
520 }
521 return Register();
522}
523
524Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
525 if (RetVT.SimpleTy != MVT::nxv4f32)
526 return Register();
527 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
528 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
529 }
530 return Register();
531}
532
533Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
534 if (RetVT.SimpleTy != MVT::nxv2f64)
535 return Register();
536 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
537 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
538 }
539 return Register();
540}
541
542Register fastEmit_AArch64ISD_FRSQRTE_r(MVT VT, MVT RetVT, Register Op0) {
543 switch (VT.SimpleTy) {
544 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(RetVT, Op0);
545 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(RetVT, Op0);
546 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(RetVT, Op0);
547 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(RetVT, Op0);
548 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(RetVT, Op0);
549 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(RetVT, Op0);
550 default: return Register();
551 }
552}
553
554// FastEmit functions for AArch64ISD::PROBED_ALLOCA.
555
556Register fastEmit_AArch64ISD_PROBED_ALLOCA_MVT_i64_r(MVT RetVT, Register Op0) {
557 if (RetVT.SimpleTy != MVT::isVoid)
558 return Register();
559 return fastEmitInst_r(MachineInstOpcode: AArch64::PROBED_STACKALLOC_DYN, RC: &AArch64::GPR64commonRegClass, Op0);
560}
561
562Register fastEmit_AArch64ISD_PROBED_ALLOCA_r(MVT VT, MVT RetVT, Register Op0) {
563 switch (VT.SimpleTy) {
564 case MVT::i64: return fastEmit_AArch64ISD_PROBED_ALLOCA_MVT_i64_r(RetVT, Op0);
565 default: return Register();
566 }
567}
568
569// FastEmit functions for AArch64ISD::REV16.
570
571Register fastEmit_AArch64ISD_REV16_MVT_i32_r(MVT RetVT, Register Op0) {
572 if (RetVT.SimpleTy != MVT::i32)
573 return Register();
574 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16Wr, RC: &AArch64::GPR32RegClass, Op0);
575}
576
577Register fastEmit_AArch64ISD_REV16_MVT_i64_r(MVT RetVT, Register Op0) {
578 if (RetVT.SimpleTy != MVT::i64)
579 return Register();
580 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16Xr, RC: &AArch64::GPR64RegClass, Op0);
581}
582
583Register fastEmit_AArch64ISD_REV16_MVT_v8i8_r(MVT RetVT, Register Op0) {
584 if (RetVT.SimpleTy != MVT::v8i8)
585 return Register();
586 if ((Subtarget->isNeonAvailable())) {
587 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
588 }
589 return Register();
590}
591
592Register fastEmit_AArch64ISD_REV16_MVT_v16i8_r(MVT RetVT, Register Op0) {
593 if (RetVT.SimpleTy != MVT::v16i8)
594 return Register();
595 if ((Subtarget->isNeonAvailable())) {
596 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
597 }
598 return Register();
599}
600
601Register fastEmit_AArch64ISD_REV16_r(MVT VT, MVT RetVT, Register Op0) {
602 switch (VT.SimpleTy) {
603 case MVT::i32: return fastEmit_AArch64ISD_REV16_MVT_i32_r(RetVT, Op0);
604 case MVT::i64: return fastEmit_AArch64ISD_REV16_MVT_i64_r(RetVT, Op0);
605 case MVT::v8i8: return fastEmit_AArch64ISD_REV16_MVT_v8i8_r(RetVT, Op0);
606 case MVT::v16i8: return fastEmit_AArch64ISD_REV16_MVT_v16i8_r(RetVT, Op0);
607 default: return Register();
608 }
609}
610
611// FastEmit functions for AArch64ISD::REV32.
612
613Register fastEmit_AArch64ISD_REV32_MVT_v8i8_r(MVT RetVT, Register Op0) {
614 if (RetVT.SimpleTy != MVT::v8i8)
615 return Register();
616 if ((Subtarget->isNeonAvailable())) {
617 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
618 }
619 return Register();
620}
621
622Register fastEmit_AArch64ISD_REV32_MVT_v16i8_r(MVT RetVT, Register Op0) {
623 if (RetVT.SimpleTy != MVT::v16i8)
624 return Register();
625 if ((Subtarget->isNeonAvailable())) {
626 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
627 }
628 return Register();
629}
630
631Register fastEmit_AArch64ISD_REV32_MVT_v4i16_r(MVT RetVT, Register Op0) {
632 if (RetVT.SimpleTy != MVT::v4i16)
633 return Register();
634 if ((Subtarget->isNeonAvailable())) {
635 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
636 }
637 return Register();
638}
639
640Register fastEmit_AArch64ISD_REV32_MVT_v8i16_r(MVT RetVT, Register Op0) {
641 if (RetVT.SimpleTy != MVT::v8i16)
642 return Register();
643 if ((Subtarget->isNeonAvailable())) {
644 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
645 }
646 return Register();
647}
648
649Register fastEmit_AArch64ISD_REV32_MVT_v4f16_r(MVT RetVT, Register Op0) {
650 if (RetVT.SimpleTy != MVT::v4f16)
651 return Register();
652 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
653}
654
655Register fastEmit_AArch64ISD_REV32_MVT_v8f16_r(MVT RetVT, Register Op0) {
656 if (RetVT.SimpleTy != MVT::v8f16)
657 return Register();
658 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
659}
660
661Register fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(MVT RetVT, Register Op0) {
662 if (RetVT.SimpleTy != MVT::v4bf16)
663 return Register();
664 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
665}
666
667Register fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(MVT RetVT, Register Op0) {
668 if (RetVT.SimpleTy != MVT::v8bf16)
669 return Register();
670 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
671}
672
673Register fastEmit_AArch64ISD_REV32_r(MVT VT, MVT RetVT, Register Op0) {
674 switch (VT.SimpleTy) {
675 case MVT::v8i8: return fastEmit_AArch64ISD_REV32_MVT_v8i8_r(RetVT, Op0);
676 case MVT::v16i8: return fastEmit_AArch64ISD_REV32_MVT_v16i8_r(RetVT, Op0);
677 case MVT::v4i16: return fastEmit_AArch64ISD_REV32_MVT_v4i16_r(RetVT, Op0);
678 case MVT::v8i16: return fastEmit_AArch64ISD_REV32_MVT_v8i16_r(RetVT, Op0);
679 case MVT::v4f16: return fastEmit_AArch64ISD_REV32_MVT_v4f16_r(RetVT, Op0);
680 case MVT::v8f16: return fastEmit_AArch64ISD_REV32_MVT_v8f16_r(RetVT, Op0);
681 case MVT::v4bf16: return fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(RetVT, Op0);
682 case MVT::v8bf16: return fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(RetVT, Op0);
683 default: return Register();
684 }
685}
686
687// FastEmit functions for AArch64ISD::REV64.
688
689Register fastEmit_AArch64ISD_REV64_MVT_v8i8_r(MVT RetVT, Register Op0) {
690 if (RetVT.SimpleTy != MVT::v8i8)
691 return Register();
692 if ((Subtarget->isNeonAvailable())) {
693 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
694 }
695 return Register();
696}
697
698Register fastEmit_AArch64ISD_REV64_MVT_v16i8_r(MVT RetVT, Register Op0) {
699 if (RetVT.SimpleTy != MVT::v16i8)
700 return Register();
701 if ((Subtarget->isNeonAvailable())) {
702 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
703 }
704 return Register();
705}
706
707Register fastEmit_AArch64ISD_REV64_MVT_v4i16_r(MVT RetVT, Register Op0) {
708 if (RetVT.SimpleTy != MVT::v4i16)
709 return Register();
710 if ((Subtarget->isNeonAvailable())) {
711 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
712 }
713 return Register();
714}
715
716Register fastEmit_AArch64ISD_REV64_MVT_v8i16_r(MVT RetVT, Register Op0) {
717 if (RetVT.SimpleTy != MVT::v8i16)
718 return Register();
719 if ((Subtarget->isNeonAvailable())) {
720 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
721 }
722 return Register();
723}
724
725Register fastEmit_AArch64ISD_REV64_MVT_v2i32_r(MVT RetVT, Register Op0) {
726 if (RetVT.SimpleTy != MVT::v2i32)
727 return Register();
728 if ((Subtarget->isNeonAvailable())) {
729 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
730 }
731 return Register();
732}
733
734Register fastEmit_AArch64ISD_REV64_MVT_v4i32_r(MVT RetVT, Register Op0) {
735 if (RetVT.SimpleTy != MVT::v4i32)
736 return Register();
737 if ((Subtarget->isNeonAvailable())) {
738 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
739 }
740 return Register();
741}
742
743Register fastEmit_AArch64ISD_REV64_MVT_v4f16_r(MVT RetVT, Register Op0) {
744 if (RetVT.SimpleTy != MVT::v4f16)
745 return Register();
746 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
747}
748
749Register fastEmit_AArch64ISD_REV64_MVT_v8f16_r(MVT RetVT, Register Op0) {
750 if (RetVT.SimpleTy != MVT::v8f16)
751 return Register();
752 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
753}
754
755Register fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(MVT RetVT, Register Op0) {
756 if (RetVT.SimpleTy != MVT::v4bf16)
757 return Register();
758 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
759}
760
761Register fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(MVT RetVT, Register Op0) {
762 if (RetVT.SimpleTy != MVT::v8bf16)
763 return Register();
764 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
765}
766
767Register fastEmit_AArch64ISD_REV64_MVT_v2f32_r(MVT RetVT, Register Op0) {
768 if (RetVT.SimpleTy != MVT::v2f32)
769 return Register();
770 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
771}
772
773Register fastEmit_AArch64ISD_REV64_MVT_v4f32_r(MVT RetVT, Register Op0) {
774 if (RetVT.SimpleTy != MVT::v4f32)
775 return Register();
776 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
777}
778
779Register fastEmit_AArch64ISD_REV64_r(MVT VT, MVT RetVT, Register Op0) {
780 switch (VT.SimpleTy) {
781 case MVT::v8i8: return fastEmit_AArch64ISD_REV64_MVT_v8i8_r(RetVT, Op0);
782 case MVT::v16i8: return fastEmit_AArch64ISD_REV64_MVT_v16i8_r(RetVT, Op0);
783 case MVT::v4i16: return fastEmit_AArch64ISD_REV64_MVT_v4i16_r(RetVT, Op0);
784 case MVT::v8i16: return fastEmit_AArch64ISD_REV64_MVT_v8i16_r(RetVT, Op0);
785 case MVT::v2i32: return fastEmit_AArch64ISD_REV64_MVT_v2i32_r(RetVT, Op0);
786 case MVT::v4i32: return fastEmit_AArch64ISD_REV64_MVT_v4i32_r(RetVT, Op0);
787 case MVT::v4f16: return fastEmit_AArch64ISD_REV64_MVT_v4f16_r(RetVT, Op0);
788 case MVT::v8f16: return fastEmit_AArch64ISD_REV64_MVT_v8f16_r(RetVT, Op0);
789 case MVT::v4bf16: return fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(RetVT, Op0);
790 case MVT::v8bf16: return fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(RetVT, Op0);
791 case MVT::v2f32: return fastEmit_AArch64ISD_REV64_MVT_v2f32_r(RetVT, Op0);
792 case MVT::v4f32: return fastEmit_AArch64ISD_REV64_MVT_v4f32_r(RetVT, Op0);
793 default: return Register();
794 }
795}
796
797// FastEmit functions for AArch64ISD::SADDLP.
798
799Register fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(MVT RetVT, Register Op0) {
800 if (RetVT.SimpleTy != MVT::v4i16)
801 return Register();
802 if ((Subtarget->isNeonAvailable())) {
803 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv8i8_v4i16, RC: &AArch64::FPR64RegClass, Op0);
804 }
805 return Register();
806}
807
808Register fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(MVT RetVT, Register Op0) {
809 if (RetVT.SimpleTy != MVT::v8i16)
810 return Register();
811 if ((Subtarget->isNeonAvailable())) {
812 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv16i8_v8i16, RC: &AArch64::FPR128RegClass, Op0);
813 }
814 return Register();
815}
816
817Register fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(MVT RetVT, Register Op0) {
818 if (RetVT.SimpleTy != MVT::v2i32)
819 return Register();
820 if ((Subtarget->isNeonAvailable())) {
821 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv4i16_v2i32, RC: &AArch64::FPR64RegClass, Op0);
822 }
823 return Register();
824}
825
826Register fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(MVT RetVT, Register Op0) {
827 if (RetVT.SimpleTy != MVT::v4i32)
828 return Register();
829 if ((Subtarget->isNeonAvailable())) {
830 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv8i16_v4i32, RC: &AArch64::FPR128RegClass, Op0);
831 }
832 return Register();
833}
834
835Register fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(MVT RetVT, Register Op0) {
836 if (RetVT.SimpleTy != MVT::v1i64)
837 return Register();
838 if ((Subtarget->isNeonAvailable())) {
839 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv2i32_v1i64, RC: &AArch64::FPR64RegClass, Op0);
840 }
841 return Register();
842}
843
844Register fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(MVT RetVT, Register Op0) {
845 if (RetVT.SimpleTy != MVT::v2i64)
846 return Register();
847 if ((Subtarget->isNeonAvailable())) {
848 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv4i32_v2i64, RC: &AArch64::FPR128RegClass, Op0);
849 }
850 return Register();
851}
852
853Register fastEmit_AArch64ISD_SADDLP_r(MVT VT, MVT RetVT, Register Op0) {
854 switch (VT.SimpleTy) {
855 case MVT::v8i8: return fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(RetVT, Op0);
856 case MVT::v16i8: return fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(RetVT, Op0);
857 case MVT::v4i16: return fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(RetVT, Op0);
858 case MVT::v8i16: return fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(RetVT, Op0);
859 case MVT::v2i32: return fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(RetVT, Op0);
860 case MVT::v4i32: return fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(RetVT, Op0);
861 default: return Register();
862 }
863}
864
865// FastEmit functions for AArch64ISD::SITOF.
866
867Register fastEmit_AArch64ISD_SITOF_MVT_f16_r(MVT RetVT, Register Op0) {
868 if (RetVT.SimpleTy != MVT::f16)
869 return Register();
870 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
871 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i16, RC: &AArch64::FPR16RegClass, Op0);
872 }
873 return Register();
874}
875
876Register fastEmit_AArch64ISD_SITOF_MVT_f32_r(MVT RetVT, Register Op0) {
877 if (RetVT.SimpleTy != MVT::f32)
878 return Register();
879 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
880 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i32, RC: &AArch64::FPR32RegClass, Op0);
881 }
882 return Register();
883}
884
885Register fastEmit_AArch64ISD_SITOF_MVT_f64_r(MVT RetVT, Register Op0) {
886 if (RetVT.SimpleTy != MVT::f64)
887 return Register();
888 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
889 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i64, RC: &AArch64::FPR64RegClass, Op0);
890 }
891 return Register();
892}
893
894Register fastEmit_AArch64ISD_SITOF_r(MVT VT, MVT RetVT, Register Op0) {
895 switch (VT.SimpleTy) {
896 case MVT::f16: return fastEmit_AArch64ISD_SITOF_MVT_f16_r(RetVT, Op0);
897 case MVT::f32: return fastEmit_AArch64ISD_SITOF_MVT_f32_r(RetVT, Op0);
898 case MVT::f64: return fastEmit_AArch64ISD_SITOF_MVT_f64_r(RetVT, Op0);
899 default: return Register();
900 }
901}
902
903// FastEmit functions for AArch64ISD::SQABS.
904
905Register fastEmit_AArch64ISD_SQABS_MVT_f32_r(MVT RetVT, Register Op0) {
906 if (RetVT.SimpleTy != MVT::f32)
907 return Register();
908 if ((Subtarget->isNeonAvailable())) {
909 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv1i32, RC: &AArch64::FPR32RegClass, Op0);
910 }
911 return Register();
912}
913
914Register fastEmit_AArch64ISD_SQABS_MVT_f64_r(MVT RetVT, Register Op0) {
915 if (RetVT.SimpleTy != MVT::f64)
916 return Register();
917 if ((Subtarget->isNeonAvailable())) {
918 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
919 }
920 return Register();
921}
922
923Register fastEmit_AArch64ISD_SQABS_r(MVT VT, MVT RetVT, Register Op0) {
924 switch (VT.SimpleTy) {
925 case MVT::f32: return fastEmit_AArch64ISD_SQABS_MVT_f32_r(RetVT, Op0);
926 case MVT::f64: return fastEmit_AArch64ISD_SQABS_MVT_f64_r(RetVT, Op0);
927 default: return Register();
928 }
929}
930
931// FastEmit functions for AArch64ISD::SQNEG.
932
933Register fastEmit_AArch64ISD_SQNEG_MVT_f32_r(MVT RetVT, Register Op0) {
934 if (RetVT.SimpleTy != MVT::f32)
935 return Register();
936 if ((Subtarget->isNeonAvailable())) {
937 return fastEmitInst_r(MachineInstOpcode: AArch64::SQNEGv1i32, RC: &AArch64::FPR32RegClass, Op0);
938 }
939 return Register();
940}
941
942Register fastEmit_AArch64ISD_SQNEG_MVT_f64_r(MVT RetVT, Register Op0) {
943 if (RetVT.SimpleTy != MVT::f64)
944 return Register();
945 if ((Subtarget->isNeonAvailable())) {
946 return fastEmitInst_r(MachineInstOpcode: AArch64::SQNEGv1i64, RC: &AArch64::FPR64RegClass, Op0);
947 }
948 return Register();
949}
950
951Register fastEmit_AArch64ISD_SQNEG_r(MVT VT, MVT RetVT, Register Op0) {
952 switch (VT.SimpleTy) {
953 case MVT::f32: return fastEmit_AArch64ISD_SQNEG_MVT_f32_r(RetVT, Op0);
954 case MVT::f64: return fastEmit_AArch64ISD_SQNEG_MVT_f64_r(RetVT, Op0);
955 default: return Register();
956 }
957}
958
959// FastEmit functions for AArch64ISD::SUNPKHI.
960
961Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
962 if (RetVT.SimpleTy != MVT::nxv8i16)
963 return Register();
964 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
965 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
966 }
967 return Register();
968}
969
970Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
971 if (RetVT.SimpleTy != MVT::nxv4i32)
972 return Register();
973 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
974 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
975 }
976 return Register();
977}
978
979Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
980 if (RetVT.SimpleTy != MVT::nxv2i64)
981 return Register();
982 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
983 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
984 }
985 return Register();
986}
987
988Register fastEmit_AArch64ISD_SUNPKHI_r(MVT VT, MVT RetVT, Register Op0) {
989 switch (VT.SimpleTy) {
990 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(RetVT, Op0);
991 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(RetVT, Op0);
992 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(RetVT, Op0);
993 default: return Register();
994 }
995}
996
997// FastEmit functions for AArch64ISD::SUNPKLO.
998
999Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
1000 if (RetVT.SimpleTy != MVT::nxv8i16)
1001 return Register();
1002 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1003 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1004 }
1005 return Register();
1006}
1007
1008Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
1009 if (RetVT.SimpleTy != MVT::nxv4i32)
1010 return Register();
1011 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1012 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1013 }
1014 return Register();
1015}
1016
1017Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
1018 if (RetVT.SimpleTy != MVT::nxv2i64)
1019 return Register();
1020 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1021 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1022 }
1023 return Register();
1024}
1025
1026Register fastEmit_AArch64ISD_SUNPKLO_r(MVT VT, MVT RetVT, Register Op0) {
1027 switch (VT.SimpleTy) {
1028 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(RetVT, Op0);
1029 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(RetVT, Op0);
1030 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(RetVT, Op0);
1031 default: return Register();
1032 }
1033}
1034
1035// FastEmit functions for AArch64ISD::UADDLP.
1036
1037Register fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(MVT RetVT, Register Op0) {
1038 if (RetVT.SimpleTy != MVT::v4i16)
1039 return Register();
1040 if ((Subtarget->isNeonAvailable())) {
1041 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv8i8_v4i16, RC: &AArch64::FPR64RegClass, Op0);
1042 }
1043 return Register();
1044}
1045
1046Register fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(MVT RetVT, Register Op0) {
1047 if (RetVT.SimpleTy != MVT::v8i16)
1048 return Register();
1049 if ((Subtarget->isNeonAvailable())) {
1050 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv16i8_v8i16, RC: &AArch64::FPR128RegClass, Op0);
1051 }
1052 return Register();
1053}
1054
1055Register fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(MVT RetVT, Register Op0) {
1056 if (RetVT.SimpleTy != MVT::v2i32)
1057 return Register();
1058 if ((Subtarget->isNeonAvailable())) {
1059 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv4i16_v2i32, RC: &AArch64::FPR64RegClass, Op0);
1060 }
1061 return Register();
1062}
1063
1064Register fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(MVT RetVT, Register Op0) {
1065 if (RetVT.SimpleTy != MVT::v4i32)
1066 return Register();
1067 if ((Subtarget->isNeonAvailable())) {
1068 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv8i16_v4i32, RC: &AArch64::FPR128RegClass, Op0);
1069 }
1070 return Register();
1071}
1072
1073Register fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(MVT RetVT, Register Op0) {
1074 if (RetVT.SimpleTy != MVT::v1i64)
1075 return Register();
1076 if ((Subtarget->isNeonAvailable())) {
1077 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv2i32_v1i64, RC: &AArch64::FPR64RegClass, Op0);
1078 }
1079 return Register();
1080}
1081
1082Register fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(MVT RetVT, Register Op0) {
1083 if (RetVT.SimpleTy != MVT::v2i64)
1084 return Register();
1085 if ((Subtarget->isNeonAvailable())) {
1086 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv4i32_v2i64, RC: &AArch64::FPR128RegClass, Op0);
1087 }
1088 return Register();
1089}
1090
1091Register fastEmit_AArch64ISD_UADDLP_r(MVT VT, MVT RetVT, Register Op0) {
1092 switch (VT.SimpleTy) {
1093 case MVT::v8i8: return fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(RetVT, Op0);
1094 case MVT::v16i8: return fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(RetVT, Op0);
1095 case MVT::v4i16: return fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(RetVT, Op0);
1096 case MVT::v8i16: return fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(RetVT, Op0);
1097 case MVT::v2i32: return fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(RetVT, Op0);
1098 case MVT::v4i32: return fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(RetVT, Op0);
1099 default: return Register();
1100 }
1101}
1102
1103// FastEmit functions for AArch64ISD::UITOF.
1104
1105Register fastEmit_AArch64ISD_UITOF_MVT_f16_r(MVT RetVT, Register Op0) {
1106 if (RetVT.SimpleTy != MVT::f16)
1107 return Register();
1108 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
1109 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i16, RC: &AArch64::FPR16RegClass, Op0);
1110 }
1111 return Register();
1112}
1113
1114Register fastEmit_AArch64ISD_UITOF_MVT_f32_r(MVT RetVT, Register Op0) {
1115 if (RetVT.SimpleTy != MVT::f32)
1116 return Register();
1117 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
1118 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i32, RC: &AArch64::FPR32RegClass, Op0);
1119 }
1120 return Register();
1121}
1122
1123Register fastEmit_AArch64ISD_UITOF_MVT_f64_r(MVT RetVT, Register Op0) {
1124 if (RetVT.SimpleTy != MVT::f64)
1125 return Register();
1126 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
1127 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i64, RC: &AArch64::FPR64RegClass, Op0);
1128 }
1129 return Register();
1130}
1131
1132Register fastEmit_AArch64ISD_UITOF_r(MVT VT, MVT RetVT, Register Op0) {
1133 switch (VT.SimpleTy) {
1134 case MVT::f16: return fastEmit_AArch64ISD_UITOF_MVT_f16_r(RetVT, Op0);
1135 case MVT::f32: return fastEmit_AArch64ISD_UITOF_MVT_f32_r(RetVT, Op0);
1136 case MVT::f64: return fastEmit_AArch64ISD_UITOF_MVT_f64_r(RetVT, Op0);
1137 default: return Register();
1138 }
1139}
1140
1141// FastEmit functions for AArch64ISD::UUNPKHI.
1142
1143Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
1144 if (RetVT.SimpleTy != MVT::nxv8i16)
1145 return Register();
1146 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1147 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1148 }
1149 return Register();
1150}
1151
1152Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
1153 if (RetVT.SimpleTy != MVT::nxv4i32)
1154 return Register();
1155 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1156 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1157 }
1158 return Register();
1159}
1160
1161Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
1162 if (RetVT.SimpleTy != MVT::nxv2i64)
1163 return Register();
1164 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1165 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1166 }
1167 return Register();
1168}
1169
1170Register fastEmit_AArch64ISD_UUNPKHI_r(MVT VT, MVT RetVT, Register Op0) {
1171 switch (VT.SimpleTy) {
1172 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(RetVT, Op0);
1173 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(RetVT, Op0);
1174 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(RetVT, Op0);
1175 default: return Register();
1176 }
1177}
1178
1179// FastEmit functions for AArch64ISD::UUNPKLO.
1180
1181Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
1182 if (RetVT.SimpleTy != MVT::nxv8i16)
1183 return Register();
1184 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1185 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1186 }
1187 return Register();
1188}
1189
1190Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
1191 if (RetVT.SimpleTy != MVT::nxv4i32)
1192 return Register();
1193 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1194 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1195 }
1196 return Register();
1197}
1198
1199Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
1200 if (RetVT.SimpleTy != MVT::nxv2i64)
1201 return Register();
1202 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1203 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1204 }
1205 return Register();
1206}
1207
1208Register fastEmit_AArch64ISD_UUNPKLO_r(MVT VT, MVT RetVT, Register Op0) {
1209 switch (VT.SimpleTy) {
1210 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(RetVT, Op0);
1211 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(RetVT, Op0);
1212 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(RetVT, Op0);
1213 default: return Register();
1214 }
1215}
1216
1217// FastEmit functions for ISD::ABS.
1218
1219Register fastEmit_ISD_ABS_MVT_i32_r(MVT RetVT, Register Op0) {
1220 if (RetVT.SimpleTy != MVT::i32)
1221 return Register();
1222 if ((Subtarget->hasCSSC())) {
1223 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSWr, RC: &AArch64::GPR32RegClass, Op0);
1224 }
1225 return Register();
1226}
1227
1228Register fastEmit_ISD_ABS_MVT_i64_r(MVT RetVT, Register Op0) {
1229 if (RetVT.SimpleTy != MVT::i64)
1230 return Register();
1231 if ((!Subtarget->hasCSSC())) {
1232 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
1233 }
1234 if ((Subtarget->hasCSSC())) {
1235 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSXr, RC: &AArch64::GPR64RegClass, Op0);
1236 }
1237 return Register();
1238}
1239
1240Register fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, Register Op0) {
1241 if (RetVT.SimpleTy != MVT::v8i8)
1242 return Register();
1243 if ((Subtarget->isNeonAvailable())) {
1244 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv8i8, RC: &AArch64::FPR64RegClass, Op0);
1245 }
1246 return Register();
1247}
1248
1249Register fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, Register Op0) {
1250 if (RetVT.SimpleTy != MVT::v16i8)
1251 return Register();
1252 if ((Subtarget->isNeonAvailable())) {
1253 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv16i8, RC: &AArch64::FPR128RegClass, Op0);
1254 }
1255 return Register();
1256}
1257
1258Register fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, Register Op0) {
1259 if (RetVT.SimpleTy != MVT::v4i16)
1260 return Register();
1261 if ((Subtarget->isNeonAvailable())) {
1262 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv4i16, RC: &AArch64::FPR64RegClass, Op0);
1263 }
1264 return Register();
1265}
1266
1267Register fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, Register Op0) {
1268 if (RetVT.SimpleTy != MVT::v8i16)
1269 return Register();
1270 if ((Subtarget->isNeonAvailable())) {
1271 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv8i16, RC: &AArch64::FPR128RegClass, Op0);
1272 }
1273 return Register();
1274}
1275
1276Register fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, Register Op0) {
1277 if (RetVT.SimpleTy != MVT::v2i32)
1278 return Register();
1279 if ((Subtarget->isNeonAvailable())) {
1280 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv2i32, RC: &AArch64::FPR64RegClass, Op0);
1281 }
1282 return Register();
1283}
1284
1285Register fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, Register Op0) {
1286 if (RetVT.SimpleTy != MVT::v4i32)
1287 return Register();
1288 if ((Subtarget->isNeonAvailable())) {
1289 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv4i32, RC: &AArch64::FPR128RegClass, Op0);
1290 }
1291 return Register();
1292}
1293
1294Register fastEmit_ISD_ABS_MVT_v1i64_r(MVT RetVT, Register Op0) {
1295 if (RetVT.SimpleTy != MVT::v1i64)
1296 return Register();
1297 if ((Subtarget->isNeonAvailable())) {
1298 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
1299 }
1300 return Register();
1301}
1302
1303Register fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, Register Op0) {
1304 if (RetVT.SimpleTy != MVT::v2i64)
1305 return Register();
1306 if ((Subtarget->isNeonAvailable())) {
1307 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv2i64, RC: &AArch64::FPR128RegClass, Op0);
1308 }
1309 return Register();
1310}
1311
1312Register fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, Register Op0) {
1313 switch (VT.SimpleTy) {
1314 case MVT::i32: return fastEmit_ISD_ABS_MVT_i32_r(RetVT, Op0);
1315 case MVT::i64: return fastEmit_ISD_ABS_MVT_i64_r(RetVT, Op0);
1316 case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0);
1317 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
1318 case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0);
1319 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
1320 case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0);
1321 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
1322 case MVT::v1i64: return fastEmit_ISD_ABS_MVT_v1i64_r(RetVT, Op0);
1323 case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0);
1324 default: return Register();
1325 }
1326}
1327
1328// FastEmit functions for ISD::BITCAST.
1329
1330Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Register Op0) {
1331 if ((!Subtarget->isLittleEndian())) {
1332 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1333 }
1334 return Register();
1335}
1336
1337Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Register Op0) {
1338 if ((!Subtarget->isLittleEndian())) {
1339 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1340 }
1341 return Register();
1342}
1343
1344Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Register Op0) {
1345 if ((!Subtarget->isLittleEndian())) {
1346 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1347 }
1348 return Register();
1349}
1350
1351Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Register Op0) {
1352 if ((!Subtarget->isLittleEndian())) {
1353 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1354 }
1355 return Register();
1356}
1357
1358Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Register Op0) {
1359 if ((!Subtarget->isLittleEndian())) {
1360 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1361 }
1362 return Register();
1363}
1364
1365Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Register Op0) {
1366 if ((!Subtarget->isLittleEndian())) {
1367 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1368 }
1369 return Register();
1370}
1371
1372Register fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, Register Op0) {
1373switch (RetVT.SimpleTy) {
1374 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0);
1375 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0);
1376 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0);
1377 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0);
1378 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Op0);
1379 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0);
1380 default: return Register();
1381}
1382}
1383
1384Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Register Op0) {
1385 if ((!Subtarget->isLittleEndian())) {
1386 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1387 }
1388 return Register();
1389}
1390
1391Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Register Op0) {
1392 if ((!Subtarget->isLittleEndian())) {
1393 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1394 }
1395 return Register();
1396}
1397
1398Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Register Op0) {
1399 if ((!Subtarget->isLittleEndian())) {
1400 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1401 }
1402 return Register();
1403}
1404
1405Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Register Op0) {
1406 if ((!Subtarget->isLittleEndian())) {
1407 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1408 }
1409 return Register();
1410}
1411
1412Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Register Op0) {
1413 if ((!Subtarget->isLittleEndian())) {
1414 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1415 }
1416 return Register();
1417}
1418
1419Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Register Op0) {
1420 if ((!Subtarget->isLittleEndian())) {
1421 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1422 }
1423 return Register();
1424}
1425
1426Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Register Op0) {
1427 if ((!Subtarget->isLittleEndian())) {
1428 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1429 }
1430 return Register();
1431}
1432
1433Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Register Op0) {
1434 if ((!Subtarget->isLittleEndian())) {
1435 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1436 }
1437 return Register();
1438}
1439
1440Register fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, Register Op0) {
1441switch (RetVT.SimpleTy) {
1442 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0);
1443 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0);
1444 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0);
1445 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0);
1446 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0);
1447 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Op0);
1448 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0);
1449 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Op0);
1450 default: return Register();
1451}
1452}
1453
1454Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Register Op0) {
1455 if ((!Subtarget->isLittleEndian())) {
1456 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1457 }
1458 return Register();
1459}
1460
1461Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Register Op0) {
1462 if ((!Subtarget->isLittleEndian())) {
1463 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1464 }
1465 return Register();
1466}
1467
1468Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Register Op0) {
1469 if ((!Subtarget->isLittleEndian())) {
1470 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1471 }
1472 return Register();
1473}
1474
1475Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Register Op0) {
1476 if ((!Subtarget->isLittleEndian())) {
1477 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1478 }
1479 return Register();
1480}
1481
1482Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Register Op0) {
1483 if ((!Subtarget->isLittleEndian())) {
1484 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1485 }
1486 return Register();
1487}
1488
1489Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Register Op0) {
1490 if ((!Subtarget->isLittleEndian())) {
1491 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1492 }
1493 return Register();
1494}
1495
1496Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Register Op0) {
1497 if ((!Subtarget->isLittleEndian())) {
1498 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1499 }
1500 return Register();
1501}
1502
1503Register fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, Register Op0) {
1504switch (RetVT.SimpleTy) {
1505 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0);
1506 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0);
1507 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0);
1508 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0);
1509 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Op0);
1510 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0);
1511 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0);
1512 default: return Register();
1513}
1514}
1515
1516Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Register Op0) {
1517 if ((!Subtarget->isLittleEndian())) {
1518 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1519 }
1520 return Register();
1521}
1522
1523Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Register Op0) {
1524 if ((!Subtarget->isLittleEndian())) {
1525 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1526 }
1527 return Register();
1528}
1529
1530Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Register Op0) {
1531 if ((!Subtarget->isLittleEndian())) {
1532 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1533 }
1534 return Register();
1535}
1536
1537Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Register Op0) {
1538 if ((!Subtarget->isLittleEndian())) {
1539 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1540 }
1541 return Register();
1542}
1543
1544Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Register Op0) {
1545 if ((!Subtarget->isLittleEndian())) {
1546 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1547 }
1548 return Register();
1549}
1550
1551Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Register Op0) {
1552 if ((!Subtarget->isLittleEndian())) {
1553 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1554 }
1555 return Register();
1556}
1557
1558Register fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, Register Op0) {
1559switch (RetVT.SimpleTy) {
1560 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0);
1561 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0);
1562 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0);
1563 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0);
1564 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0);
1565 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Op0);
1566 default: return Register();
1567}
1568}
1569
1570Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Register Op0) {
1571 if ((!Subtarget->isLittleEndian())) {
1572 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1573 }
1574 return Register();
1575}
1576
1577Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Register Op0) {
1578 if ((!Subtarget->isLittleEndian())) {
1579 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1580 }
1581 return Register();
1582}
1583
1584Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Register Op0) {
1585 if ((!Subtarget->isLittleEndian())) {
1586 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1587 }
1588 return Register();
1589}
1590
1591Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Register Op0) {
1592 if ((!Subtarget->isLittleEndian())) {
1593 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1594 }
1595 return Register();
1596}
1597
1598Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Register Op0) {
1599 if ((!Subtarget->isLittleEndian())) {
1600 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1601 }
1602 return Register();
1603}
1604
1605Register fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, Register Op0) {
1606switch (RetVT.SimpleTy) {
1607 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0);
1608 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0);
1609 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0);
1610 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0);
1611 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0);
1612 default: return Register();
1613}
1614}
1615
1616Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Register Op0) {
1617 if ((!Subtarget->isLittleEndian())) {
1618 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1619 }
1620 return Register();
1621}
1622
1623Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Register Op0) {
1624 if ((!Subtarget->isLittleEndian())) {
1625 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1626 }
1627 return Register();
1628}
1629
1630Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Register Op0) {
1631 if ((!Subtarget->isLittleEndian())) {
1632 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1633 }
1634 return Register();
1635}
1636
1637Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Register Op0) {
1638 if ((!Subtarget->isLittleEndian())) {
1639 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1640 }
1641 return Register();
1642}
1643
1644Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Register Op0) {
1645 if ((!Subtarget->isLittleEndian())) {
1646 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1647 }
1648 return Register();
1649}
1650
1651Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Register Op0) {
1652 if ((!Subtarget->isLittleEndian())) {
1653 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1654 }
1655 return Register();
1656}
1657
1658Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Register Op0) {
1659 if ((!Subtarget->isLittleEndian())) {
1660 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1661 }
1662 return Register();
1663}
1664
1665Register fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, Register Op0) {
1666switch (RetVT.SimpleTy) {
1667 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0);
1668 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0);
1669 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0);
1670 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0);
1671 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0);
1672 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Op0);
1673 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Op0);
1674 default: return Register();
1675}
1676}
1677
1678Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Register Op0) {
1679 if ((!Subtarget->isLittleEndian())) {
1680 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1681 }
1682 return Register();
1683}
1684
1685Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Register Op0) {
1686 if ((!Subtarget->isLittleEndian())) {
1687 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1688 }
1689 return Register();
1690}
1691
1692Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Register Op0) {
1693 if ((!Subtarget->isLittleEndian())) {
1694 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1695 }
1696 return Register();
1697}
1698
1699Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Register Op0) {
1700 if ((!Subtarget->isLittleEndian())) {
1701 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1702 }
1703 return Register();
1704}
1705
1706Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Register Op0) {
1707 if ((!Subtarget->isLittleEndian())) {
1708 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1709 }
1710 return Register();
1711}
1712
1713Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Register Op0) {
1714 if ((!Subtarget->isLittleEndian())) {
1715 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1716 }
1717 return Register();
1718}
1719
1720Register fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, Register Op0) {
1721switch (RetVT.SimpleTy) {
1722 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0);
1723 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0);
1724 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0);
1725 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0);
1726 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Op0);
1727 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0);
1728 default: return Register();
1729}
1730}
1731
1732Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Register Op0) {
1733 if ((!Subtarget->isLittleEndian())) {
1734 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1735 }
1736 return Register();
1737}
1738
1739Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Register Op0) {
1740 if ((!Subtarget->isLittleEndian())) {
1741 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1742 }
1743 return Register();
1744}
1745
1746Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Register Op0) {
1747 if ((!Subtarget->isLittleEndian())) {
1748 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1749 }
1750 return Register();
1751}
1752
1753Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Register Op0) {
1754 if ((!Subtarget->isLittleEndian())) {
1755 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1756 }
1757 return Register();
1758}
1759
1760Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Register Op0) {
1761 if ((!Subtarget->isLittleEndian())) {
1762 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1763 }
1764 return Register();
1765}
1766
1767Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Register Op0) {
1768 if ((!Subtarget->isLittleEndian())) {
1769 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1770 }
1771 return Register();
1772}
1773
1774Register fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, Register Op0) {
1775switch (RetVT.SimpleTy) {
1776 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0);
1777 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0);
1778 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0);
1779 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0);
1780 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Op0);
1781 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0);
1782 default: return Register();
1783}
1784}
1785
1786Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Register Op0) {
1787 if ((!Subtarget->isLittleEndian())) {
1788 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1789 }
1790 return Register();
1791}
1792
1793Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Register Op0) {
1794 if ((!Subtarget->isLittleEndian())) {
1795 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1796 }
1797 return Register();
1798}
1799
1800Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Register Op0) {
1801 if ((!Subtarget->isLittleEndian())) {
1802 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1803 }
1804 return Register();
1805}
1806
1807Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Register Op0) {
1808 if ((!Subtarget->isLittleEndian())) {
1809 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1810 }
1811 return Register();
1812}
1813
1814Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Register Op0) {
1815 if ((!Subtarget->isLittleEndian())) {
1816 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1817 }
1818 return Register();
1819}
1820
1821Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Register Op0) {
1822 if ((!Subtarget->isLittleEndian())) {
1823 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1824 }
1825 return Register();
1826}
1827
1828Register fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, Register Op0) {
1829switch (RetVT.SimpleTy) {
1830 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0);
1831 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0);
1832 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0);
1833 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0);
1834 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Op0);
1835 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0);
1836 default: return Register();
1837}
1838}
1839
1840Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Register Op0) {
1841 if ((!Subtarget->isLittleEndian())) {
1842 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1843 }
1844 return Register();
1845}
1846
1847Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Register Op0) {
1848 if ((!Subtarget->isLittleEndian())) {
1849 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1850 }
1851 return Register();
1852}
1853
1854Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Register Op0) {
1855 if ((!Subtarget->isLittleEndian())) {
1856 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1857 }
1858 return Register();
1859}
1860
1861Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Register Op0) {
1862 if ((!Subtarget->isLittleEndian())) {
1863 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1864 }
1865 return Register();
1866}
1867
1868Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Register Op0) {
1869 if ((!Subtarget->isLittleEndian())) {
1870 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1871 }
1872 return Register();
1873}
1874
1875Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Register Op0) {
1876 if ((!Subtarget->isLittleEndian())) {
1877 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1878 }
1879 return Register();
1880}
1881
1882Register fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, Register Op0) {
1883switch (RetVT.SimpleTy) {
1884 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0);
1885 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0);
1886 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0);
1887 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0);
1888 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0);
1889 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Op0);
1890 default: return Register();
1891}
1892}
1893
1894Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Register Op0) {
1895 if ((!Subtarget->isLittleEndian())) {
1896 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1897 }
1898 return Register();
1899}
1900
1901Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Register Op0) {
1902 if ((!Subtarget->isLittleEndian())) {
1903 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1904 }
1905 return Register();
1906}
1907
1908Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Register Op0) {
1909 if ((!Subtarget->isLittleEndian())) {
1910 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1911 }
1912 return Register();
1913}
1914
1915Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Register Op0) {
1916 if ((!Subtarget->isLittleEndian())) {
1917 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1918 }
1919 return Register();
1920}
1921
1922Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Register Op0) {
1923 if ((!Subtarget->isLittleEndian())) {
1924 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1925 }
1926 return Register();
1927}
1928
1929Register fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, Register Op0) {
1930switch (RetVT.SimpleTy) {
1931 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0);
1932 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0);
1933 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0);
1934 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0);
1935 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0);
1936 default: return Register();
1937}
1938}
1939
1940Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Register Op0) {
1941 if ((!Subtarget->isLittleEndian())) {
1942 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1943 }
1944 return Register();
1945}
1946
1947Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Register Op0) {
1948 if ((!Subtarget->isLittleEndian())) {
1949 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1950 }
1951 return Register();
1952}
1953
1954Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Register Op0) {
1955 if ((!Subtarget->isLittleEndian())) {
1956 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1957 }
1958 return Register();
1959}
1960
1961Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Register Op0) {
1962 if ((!Subtarget->isLittleEndian())) {
1963 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1964 }
1965 return Register();
1966}
1967
1968Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Register Op0) {
1969 if ((!Subtarget->isLittleEndian())) {
1970 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1971 }
1972 return Register();
1973}
1974
1975Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(Register Op0) {
1976 if ((!Subtarget->isLittleEndian())) {
1977 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1978 }
1979 return Register();
1980}
1981
1982Register fastEmit_ISD_BITCAST_MVT_v4bf16_r(MVT RetVT, Register Op0) {
1983switch (RetVT.SimpleTy) {
1984 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Op0);
1985 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Op0);
1986 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Op0);
1987 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Op0);
1988 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Op0);
1989 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(Op0);
1990 default: return Register();
1991}
1992}
1993
1994Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Register Op0) {
1995 if ((!Subtarget->isLittleEndian())) {
1996 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1997 }
1998 return Register();
1999}
2000
2001Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Register Op0) {
2002 if ((!Subtarget->isLittleEndian())) {
2003 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2004 }
2005 return Register();
2006}
2007
2008Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Register Op0) {
2009 if ((!Subtarget->isLittleEndian())) {
2010 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2011 }
2012 return Register();
2013}
2014
2015Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Register Op0) {
2016 if ((!Subtarget->isLittleEndian())) {
2017 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2018 }
2019 return Register();
2020}
2021
2022Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Register Op0) {
2023 if ((!Subtarget->isLittleEndian())) {
2024 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2025 }
2026 return Register();
2027}
2028
2029Register fastEmit_ISD_BITCAST_MVT_v8bf16_r(MVT RetVT, Register Op0) {
2030switch (RetVT.SimpleTy) {
2031 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Op0);
2032 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Op0);
2033 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Op0);
2034 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Op0);
2035 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Op0);
2036 default: return Register();
2037}
2038}
2039
2040Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Register Op0) {
2041 if ((!Subtarget->isLittleEndian())) {
2042 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2043 }
2044 return Register();
2045}
2046
2047Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Register Op0) {
2048 if ((!Subtarget->isLittleEndian())) {
2049 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
2050 }
2051 return Register();
2052}
2053
2054Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Register Op0) {
2055 if ((!Subtarget->isLittleEndian())) {
2056 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2057 }
2058 return Register();
2059}
2060
2061Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Register Op0) {
2062 if ((!Subtarget->isLittleEndian())) {
2063 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2064 }
2065 return Register();
2066}
2067
2068Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Register Op0) {
2069 if ((!Subtarget->isLittleEndian())) {
2070 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2071 }
2072 return Register();
2073}
2074
2075Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Register Op0) {
2076 if ((!Subtarget->isLittleEndian())) {
2077 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2078 }
2079 return Register();
2080}
2081
2082Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Register Op0) {
2083 if ((!Subtarget->isLittleEndian())) {
2084 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2085 }
2086 return Register();
2087}
2088
2089Register fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, Register Op0) {
2090switch (RetVT.SimpleTy) {
2091 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0);
2092 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0);
2093 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0);
2094 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0);
2095 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0);
2096 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Op0);
2097 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Op0);
2098 default: return Register();
2099}
2100}
2101
2102Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Register Op0) {
2103 if ((!Subtarget->isLittleEndian())) {
2104 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
2105 }
2106 return Register();
2107}
2108
2109Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Register Op0) {
2110 if ((!Subtarget->isLittleEndian())) {
2111 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2112 }
2113 return Register();
2114}
2115
2116Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Register Op0) {
2117 if ((!Subtarget->isLittleEndian())) {
2118 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2119 }
2120 return Register();
2121}
2122
2123Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Register Op0) {
2124 if ((!Subtarget->isLittleEndian())) {
2125 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2126 }
2127 return Register();
2128}
2129
2130Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Register Op0) {
2131 if ((!Subtarget->isLittleEndian())) {
2132 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2133 }
2134 return Register();
2135}
2136
2137Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Register Op0) {
2138 if ((!Subtarget->isLittleEndian())) {
2139 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2140 }
2141 return Register();
2142}
2143
2144Register fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, Register Op0) {
2145switch (RetVT.SimpleTy) {
2146 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0);
2147 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0);
2148 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0);
2149 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0);
2150 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Op0);
2151 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0);
2152 default: return Register();
2153}
2154}
2155
2156Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Register Op0) {
2157 if ((!Subtarget->isLittleEndian())) {
2158 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
2159 }
2160 return Register();
2161}
2162
2163Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Register Op0) {
2164 if ((!Subtarget->isLittleEndian())) {
2165 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2166 }
2167 return Register();
2168}
2169
2170Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Register Op0) {
2171 if ((!Subtarget->isLittleEndian())) {
2172 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2173 }
2174 return Register();
2175}
2176
2177Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Register Op0) {
2178 if ((!Subtarget->isLittleEndian())) {
2179 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2180 }
2181 return Register();
2182}
2183
2184Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(Register Op0) {
2185 if ((!Subtarget->isLittleEndian())) {
2186 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2187 }
2188 return Register();
2189}
2190
2191Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Register Op0) {
2192 if ((!Subtarget->isLittleEndian())) {
2193 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2194 }
2195 return Register();
2196}
2197
2198Register fastEmit_ISD_BITCAST_MVT_v1f64_r(MVT RetVT, Register Op0) {
2199switch (RetVT.SimpleTy) {
2200 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Op0);
2201 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Op0);
2202 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Op0);
2203 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Op0);
2204 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(Op0);
2205 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Op0);
2206 default: return Register();
2207}
2208}
2209
2210Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Register Op0) {
2211 if ((!Subtarget->isLittleEndian())) {
2212 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2213 }
2214 return Register();
2215}
2216
2217Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Register Op0) {
2218 if ((!Subtarget->isLittleEndian())) {
2219 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2220 }
2221 return Register();
2222}
2223
2224Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Register Op0) {
2225 if ((!Subtarget->isLittleEndian())) {
2226 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2227 }
2228 return Register();
2229}
2230
2231Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Register Op0) {
2232 if ((!Subtarget->isLittleEndian())) {
2233 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2234 }
2235 return Register();
2236}
2237
2238Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Register Op0) {
2239 if ((!Subtarget->isLittleEndian())) {
2240 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2241 }
2242 return Register();
2243}
2244
2245Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Register Op0) {
2246 if ((!Subtarget->isLittleEndian())) {
2247 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2248 }
2249 return Register();
2250}
2251
2252Register fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, Register Op0) {
2253switch (RetVT.SimpleTy) {
2254 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0);
2255 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0);
2256 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0);
2257 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0);
2258 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Op0);
2259 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0);
2260 default: return Register();
2261}
2262}
2263
2264Register fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, Register Op0) {
2265 switch (VT.SimpleTy) {
2266 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
2267 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0);
2268 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0);
2269 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0);
2270 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0);
2271 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0);
2272 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0);
2273 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0);
2274 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0);
2275 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0);
2276 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0);
2277 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v4bf16_r(RetVT, Op0);
2278 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v8bf16_r(RetVT, Op0);
2279 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0);
2280 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0);
2281 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v1f64_r(RetVT, Op0);
2282 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0);
2283 default: return Register();
2284 }
2285}
2286
2287// FastEmit functions for ISD::BITREVERSE.
2288
2289Register fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, Register Op0) {
2290 if (RetVT.SimpleTy != MVT::i32)
2291 return Register();
2292 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITWr, RC: &AArch64::GPR32RegClass, Op0);
2293}
2294
2295Register fastEmit_ISD_BITREVERSE_MVT_i64_r(MVT RetVT, Register Op0) {
2296 if (RetVT.SimpleTy != MVT::i64)
2297 return Register();
2298 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITXr, RC: &AArch64::GPR64RegClass, Op0);
2299}
2300
2301Register fastEmit_ISD_BITREVERSE_MVT_v8i8_r(MVT RetVT, Register Op0) {
2302 if (RetVT.SimpleTy != MVT::v8i8)
2303 return Register();
2304 if ((Subtarget->isNeonAvailable())) {
2305 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITv8i8, RC: &AArch64::FPR64RegClass, Op0);
2306 }
2307 return Register();
2308}
2309
2310Register fastEmit_ISD_BITREVERSE_MVT_v16i8_r(MVT RetVT, Register Op0) {
2311 if (RetVT.SimpleTy != MVT::v16i8)
2312 return Register();
2313 if ((Subtarget->isNeonAvailable())) {
2314 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITv16i8, RC: &AArch64::FPR128RegClass, Op0);
2315 }
2316 return Register();
2317}
2318
2319Register fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, Register Op0) {
2320 switch (VT.SimpleTy) {
2321 case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0);
2322 case MVT::i64: return fastEmit_ISD_BITREVERSE_MVT_i64_r(RetVT, Op0);
2323 case MVT::v8i8: return fastEmit_ISD_BITREVERSE_MVT_v8i8_r(RetVT, Op0);
2324 case MVT::v16i8: return fastEmit_ISD_BITREVERSE_MVT_v16i8_r(RetVT, Op0);
2325 default: return Register();
2326 }
2327}
2328
2329// FastEmit functions for ISD::BRIND.
2330
2331Register fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, Register Op0) {
2332 if (RetVT.SimpleTy != MVT::isVoid)
2333 return Register();
2334 return fastEmitInst_r(MachineInstOpcode: AArch64::BR, RC: &AArch64::GPR64RegClass, Op0);
2335}
2336
2337Register fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, Register Op0) {
2338 switch (VT.SimpleTy) {
2339 case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0);
2340 default: return Register();
2341 }
2342}
2343
2344// FastEmit functions for ISD::BSWAP.
2345
2346Register fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, Register Op0) {
2347 if (RetVT.SimpleTy != MVT::i32)
2348 return Register();
2349 return fastEmitInst_r(MachineInstOpcode: AArch64::REVWr, RC: &AArch64::GPR32RegClass, Op0);
2350}
2351
2352Register fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, Register Op0) {
2353 if (RetVT.SimpleTy != MVT::i64)
2354 return Register();
2355 return fastEmitInst_r(MachineInstOpcode: AArch64::REVXr, RC: &AArch64::GPR64RegClass, Op0);
2356}
2357
2358Register fastEmit_ISD_BSWAP_MVT_v4i16_r(MVT RetVT, Register Op0) {
2359 if (RetVT.SimpleTy != MVT::v4i16)
2360 return Register();
2361 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
2362}
2363
2364Register fastEmit_ISD_BSWAP_MVT_v8i16_r(MVT RetVT, Register Op0) {
2365 if (RetVT.SimpleTy != MVT::v8i16)
2366 return Register();
2367 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
2368}
2369
2370Register fastEmit_ISD_BSWAP_MVT_v2i32_r(MVT RetVT, Register Op0) {
2371 if (RetVT.SimpleTy != MVT::v2i32)
2372 return Register();
2373 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
2374}
2375
2376Register fastEmit_ISD_BSWAP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2377 if (RetVT.SimpleTy != MVT::v4i32)
2378 return Register();
2379 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
2380}
2381
2382Register fastEmit_ISD_BSWAP_MVT_v2i64_r(MVT RetVT, Register Op0) {
2383 if (RetVT.SimpleTy != MVT::v2i64)
2384 return Register();
2385 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2386}
2387
2388Register fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, Register Op0) {
2389 switch (VT.SimpleTy) {
2390 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
2391 case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0);
2392 case MVT::v4i16: return fastEmit_ISD_BSWAP_MVT_v4i16_r(RetVT, Op0);
2393 case MVT::v8i16: return fastEmit_ISD_BSWAP_MVT_v8i16_r(RetVT, Op0);
2394 case MVT::v2i32: return fastEmit_ISD_BSWAP_MVT_v2i32_r(RetVT, Op0);
2395 case MVT::v4i32: return fastEmit_ISD_BSWAP_MVT_v4i32_r(RetVT, Op0);
2396 case MVT::v2i64: return fastEmit_ISD_BSWAP_MVT_v2i64_r(RetVT, Op0);
2397 default: return Register();
2398 }
2399}
2400
2401// FastEmit functions for ISD::CTLS.
2402
2403Register fastEmit_ISD_CTLS_MVT_i32_r(MVT RetVT, Register Op0) {
2404 if (RetVT.SimpleTy != MVT::i32)
2405 return Register();
2406 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSWr, RC: &AArch64::GPR32RegClass, Op0);
2407}
2408
2409Register fastEmit_ISD_CTLS_MVT_i64_r(MVT RetVT, Register Op0) {
2410 if (RetVT.SimpleTy != MVT::i64)
2411 return Register();
2412 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSXr, RC: &AArch64::GPR64RegClass, Op0);
2413}
2414
2415Register fastEmit_ISD_CTLS_MVT_v8i8_r(MVT RetVT, Register Op0) {
2416 if (RetVT.SimpleTy != MVT::v8i8)
2417 return Register();
2418 if ((Subtarget->isNeonAvailable())) {
2419 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv8i8, RC: &AArch64::FPR64RegClass, Op0);
2420 }
2421 return Register();
2422}
2423
2424Register fastEmit_ISD_CTLS_MVT_v16i8_r(MVT RetVT, Register Op0) {
2425 if (RetVT.SimpleTy != MVT::v16i8)
2426 return Register();
2427 if ((Subtarget->isNeonAvailable())) {
2428 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv16i8, RC: &AArch64::FPR128RegClass, Op0);
2429 }
2430 return Register();
2431}
2432
2433Register fastEmit_ISD_CTLS_MVT_v4i16_r(MVT RetVT, Register Op0) {
2434 if (RetVT.SimpleTy != MVT::v4i16)
2435 return Register();
2436 if ((Subtarget->isNeonAvailable())) {
2437 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv4i16, RC: &AArch64::FPR64RegClass, Op0);
2438 }
2439 return Register();
2440}
2441
2442Register fastEmit_ISD_CTLS_MVT_v8i16_r(MVT RetVT, Register Op0) {
2443 if (RetVT.SimpleTy != MVT::v8i16)
2444 return Register();
2445 if ((Subtarget->isNeonAvailable())) {
2446 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv8i16, RC: &AArch64::FPR128RegClass, Op0);
2447 }
2448 return Register();
2449}
2450
2451Register fastEmit_ISD_CTLS_MVT_v2i32_r(MVT RetVT, Register Op0) {
2452 if (RetVT.SimpleTy != MVT::v2i32)
2453 return Register();
2454 if ((Subtarget->isNeonAvailable())) {
2455 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv2i32, RC: &AArch64::FPR64RegClass, Op0);
2456 }
2457 return Register();
2458}
2459
2460Register fastEmit_ISD_CTLS_MVT_v4i32_r(MVT RetVT, Register Op0) {
2461 if (RetVT.SimpleTy != MVT::v4i32)
2462 return Register();
2463 if ((Subtarget->isNeonAvailable())) {
2464 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv4i32, RC: &AArch64::FPR128RegClass, Op0);
2465 }
2466 return Register();
2467}
2468
2469Register fastEmit_ISD_CTLS_r(MVT VT, MVT RetVT, Register Op0) {
2470 switch (VT.SimpleTy) {
2471 case MVT::i32: return fastEmit_ISD_CTLS_MVT_i32_r(RetVT, Op0);
2472 case MVT::i64: return fastEmit_ISD_CTLS_MVT_i64_r(RetVT, Op0);
2473 case MVT::v8i8: return fastEmit_ISD_CTLS_MVT_v8i8_r(RetVT, Op0);
2474 case MVT::v16i8: return fastEmit_ISD_CTLS_MVT_v16i8_r(RetVT, Op0);
2475 case MVT::v4i16: return fastEmit_ISD_CTLS_MVT_v4i16_r(RetVT, Op0);
2476 case MVT::v8i16: return fastEmit_ISD_CTLS_MVT_v8i16_r(RetVT, Op0);
2477 case MVT::v2i32: return fastEmit_ISD_CTLS_MVT_v2i32_r(RetVT, Op0);
2478 case MVT::v4i32: return fastEmit_ISD_CTLS_MVT_v4i32_r(RetVT, Op0);
2479 default: return Register();
2480 }
2481}
2482
2483// FastEmit functions for ISD::CTLZ.
2484
2485Register fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, Register Op0) {
2486 if (RetVT.SimpleTy != MVT::i32)
2487 return Register();
2488 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZWr, RC: &AArch64::GPR32RegClass, Op0);
2489}
2490
2491Register fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, Register Op0) {
2492 if (RetVT.SimpleTy != MVT::i64)
2493 return Register();
2494 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZXr, RC: &AArch64::GPR64RegClass, Op0);
2495}
2496
2497Register fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, Register Op0) {
2498 if (RetVT.SimpleTy != MVT::v8i8)
2499 return Register();
2500 if ((Subtarget->isNeonAvailable())) {
2501 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv8i8, RC: &AArch64::FPR64RegClass, Op0);
2502 }
2503 return Register();
2504}
2505
2506Register fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, Register Op0) {
2507 if (RetVT.SimpleTy != MVT::v16i8)
2508 return Register();
2509 if ((Subtarget->isNeonAvailable())) {
2510 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv16i8, RC: &AArch64::FPR128RegClass, Op0);
2511 }
2512 return Register();
2513}
2514
2515Register fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, Register Op0) {
2516 if (RetVT.SimpleTy != MVT::v4i16)
2517 return Register();
2518 if ((Subtarget->isNeonAvailable())) {
2519 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv4i16, RC: &AArch64::FPR64RegClass, Op0);
2520 }
2521 return Register();
2522}
2523
2524Register fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, Register Op0) {
2525 if (RetVT.SimpleTy != MVT::v8i16)
2526 return Register();
2527 if ((Subtarget->isNeonAvailable())) {
2528 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv8i16, RC: &AArch64::FPR128RegClass, Op0);
2529 }
2530 return Register();
2531}
2532
2533Register fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, Register Op0) {
2534 if (RetVT.SimpleTy != MVT::v2i32)
2535 return Register();
2536 if ((Subtarget->isNeonAvailable())) {
2537 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv2i32, RC: &AArch64::FPR64RegClass, Op0);
2538 }
2539 return Register();
2540}
2541
2542Register fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, Register Op0) {
2543 if (RetVT.SimpleTy != MVT::v4i32)
2544 return Register();
2545 if ((Subtarget->isNeonAvailable())) {
2546 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv4i32, RC: &AArch64::FPR128RegClass, Op0);
2547 }
2548 return Register();
2549}
2550
2551Register fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, Register Op0) {
2552 switch (VT.SimpleTy) {
2553 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
2554 case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0);
2555 case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0);
2556 case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0);
2557 case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0);
2558 case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0);
2559 case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0);
2560 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
2561 default: return Register();
2562 }
2563}
2564
2565// FastEmit functions for ISD::CTPOP.
2566
2567Register fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, Register Op0) {
2568 if (RetVT.SimpleTy != MVT::i32)
2569 return Register();
2570 if ((Subtarget->hasCSSC())) {
2571 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTWr, RC: &AArch64::GPR32RegClass, Op0);
2572 }
2573 return Register();
2574}
2575
2576Register fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, Register Op0) {
2577 if (RetVT.SimpleTy != MVT::i64)
2578 return Register();
2579 if ((Subtarget->hasCSSC())) {
2580 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTXr, RC: &AArch64::GPR64RegClass, Op0);
2581 }
2582 return Register();
2583}
2584
2585Register fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, Register Op0) {
2586 if (RetVT.SimpleTy != MVT::v8i8)
2587 return Register();
2588 if ((Subtarget->isNeonAvailable())) {
2589 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTv8i8, RC: &AArch64::FPR64RegClass, Op0);
2590 }
2591 return Register();
2592}
2593
2594Register fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, Register Op0) {
2595 if (RetVT.SimpleTy != MVT::v16i8)
2596 return Register();
2597 if ((Subtarget->isNeonAvailable())) {
2598 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTv16i8, RC: &AArch64::FPR128RegClass, Op0);
2599 }
2600 return Register();
2601}
2602
2603Register fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, Register Op0) {
2604 switch (VT.SimpleTy) {
2605 case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0);
2606 case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0);
2607 case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0);
2608 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
2609 default: return Register();
2610 }
2611}
2612
2613// FastEmit functions for ISD::CTTZ.
2614
2615Register fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, Register Op0) {
2616 if (RetVT.SimpleTy != MVT::i32)
2617 return Register();
2618 if ((Subtarget->hasCSSC())) {
2619 return fastEmitInst_r(MachineInstOpcode: AArch64::CTZWr, RC: &AArch64::GPR32RegClass, Op0);
2620 }
2621 return Register();
2622}
2623
2624Register fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, Register Op0) {
2625 if (RetVT.SimpleTy != MVT::i64)
2626 return Register();
2627 if ((Subtarget->hasCSSC())) {
2628 return fastEmitInst_r(MachineInstOpcode: AArch64::CTZXr, RC: &AArch64::GPR64RegClass, Op0);
2629 }
2630 return Register();
2631}
2632
2633Register fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, Register Op0) {
2634 switch (VT.SimpleTy) {
2635 case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0);
2636 case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0);
2637 default: return Register();
2638 }
2639}
2640
2641// FastEmit functions for ISD::FABS.
2642
2643Register fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, Register Op0) {
2644 if (RetVT.SimpleTy != MVT::f16)
2645 return Register();
2646 if ((Subtarget->hasFullFP16())) {
2647 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSHr, RC: &AArch64::FPR16RegClass, Op0);
2648 }
2649 return Register();
2650}
2651
2652Register fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, Register Op0) {
2653 if (RetVT.SimpleTy != MVT::f32)
2654 return Register();
2655 if ((Subtarget->hasFPARMv8())) {
2656 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSSr, RC: &AArch64::FPR32RegClass, Op0);
2657 }
2658 return Register();
2659}
2660
2661Register fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, Register Op0) {
2662 if (RetVT.SimpleTy != MVT::f64)
2663 return Register();
2664 if ((Subtarget->hasFPARMv8())) {
2665 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSDr, RC: &AArch64::FPR64RegClass, Op0);
2666 }
2667 return Register();
2668}
2669
2670Register fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, Register Op0) {
2671 if (RetVT.SimpleTy != MVT::v4f16)
2672 return Register();
2673 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2674 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f16, RC: &AArch64::FPR64RegClass, Op0);
2675 }
2676 return Register();
2677}
2678
2679Register fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, Register Op0) {
2680 if (RetVT.SimpleTy != MVT::v8f16)
2681 return Register();
2682 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2683 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv8f16, RC: &AArch64::FPR128RegClass, Op0);
2684 }
2685 return Register();
2686}
2687
2688Register fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, Register Op0) {
2689 if (RetVT.SimpleTy != MVT::v2f32)
2690 return Register();
2691 if ((Subtarget->isNeonAvailable())) {
2692 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv2f32, RC: &AArch64::FPR64RegClass, Op0);
2693 }
2694 return Register();
2695}
2696
2697Register fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, Register Op0) {
2698 if (RetVT.SimpleTy != MVT::v4f32)
2699 return Register();
2700 if ((Subtarget->isNeonAvailable())) {
2701 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f32, RC: &AArch64::FPR128RegClass, Op0);
2702 }
2703 return Register();
2704}
2705
2706Register fastEmit_ISD_FABS_MVT_v2f64_r(MVT RetVT, Register Op0) {
2707 if (RetVT.SimpleTy != MVT::v2f64)
2708 return Register();
2709 if ((Subtarget->isNeonAvailable())) {
2710 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv2f64, RC: &AArch64::FPR128RegClass, Op0);
2711 }
2712 return Register();
2713}
2714
2715Register fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, Register Op0) {
2716 switch (VT.SimpleTy) {
2717 case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0);
2718 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
2719 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
2720 case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0);
2721 case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0);
2722 case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0);
2723 case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0);
2724 case MVT::v2f64: return fastEmit_ISD_FABS_MVT_v2f64_r(RetVT, Op0);
2725 default: return Register();
2726 }
2727}
2728
2729// FastEmit functions for ISD::FCEIL.
2730
2731Register fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
2732 if (RetVT.SimpleTy != MVT::f16)
2733 return Register();
2734 if ((Subtarget->hasFullFP16())) {
2735 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPHr, RC: &AArch64::FPR16RegClass, Op0);
2736 }
2737 return Register();
2738}
2739
2740Register fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
2741 if (RetVT.SimpleTy != MVT::f32)
2742 return Register();
2743 if ((Subtarget->hasFPARMv8())) {
2744 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPSr, RC: &AArch64::FPR32RegClass, Op0);
2745 }
2746 return Register();
2747}
2748
2749Register fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
2750 if (RetVT.SimpleTy != MVT::f64)
2751 return Register();
2752 if ((Subtarget->hasFPARMv8())) {
2753 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPDr, RC: &AArch64::FPR64RegClass, Op0);
2754 }
2755 return Register();
2756}
2757
2758Register fastEmit_ISD_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
2759 if (RetVT.SimpleTy != MVT::v4f16)
2760 return Register();
2761 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2762 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f16, RC: &AArch64::FPR64RegClass, Op0);
2763 }
2764 return Register();
2765}
2766
2767Register fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
2768 if (RetVT.SimpleTy != MVT::v8f16)
2769 return Register();
2770 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2771 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv8f16, RC: &AArch64::FPR128RegClass, Op0);
2772 }
2773 return Register();
2774}
2775
2776Register fastEmit_ISD_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
2777 if (RetVT.SimpleTy != MVT::v2f32)
2778 return Register();
2779 if ((Subtarget->isNeonAvailable())) {
2780 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f32, RC: &AArch64::FPR64RegClass, Op0);
2781 }
2782 return Register();
2783}
2784
2785Register fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
2786 if (RetVT.SimpleTy != MVT::v4f32)
2787 return Register();
2788 if ((Subtarget->isNeonAvailable())) {
2789 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f32, RC: &AArch64::FPR128RegClass, Op0);
2790 }
2791 return Register();
2792}
2793
2794Register fastEmit_ISD_FCEIL_MVT_v2f64_r(MVT RetVT, Register Op0) {
2795 if (RetVT.SimpleTy != MVT::v2f64)
2796 return Register();
2797 if ((Subtarget->isNeonAvailable())) {
2798 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f64, RC: &AArch64::FPR128RegClass, Op0);
2799 }
2800 return Register();
2801}
2802
2803Register fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
2804 switch (VT.SimpleTy) {
2805 case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0);
2806 case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0);
2807 case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0);
2808 case MVT::v4f16: return fastEmit_ISD_FCEIL_MVT_v4f16_r(RetVT, Op0);
2809 case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0);
2810 case MVT::v2f32: return fastEmit_ISD_FCEIL_MVT_v2f32_r(RetVT, Op0);
2811 case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0);
2812 case MVT::v2f64: return fastEmit_ISD_FCEIL_MVT_v2f64_r(RetVT, Op0);
2813 default: return Register();
2814 }
2815}
2816
2817// FastEmit functions for ISD::FFLOOR.
2818
2819Register fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
2820 if (RetVT.SimpleTy != MVT::f16)
2821 return Register();
2822 if ((Subtarget->hasFullFP16())) {
2823 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMHr, RC: &AArch64::FPR16RegClass, Op0);
2824 }
2825 return Register();
2826}
2827
2828Register fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
2829 if (RetVT.SimpleTy != MVT::f32)
2830 return Register();
2831 if ((Subtarget->hasFPARMv8())) {
2832 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMSr, RC: &AArch64::FPR32RegClass, Op0);
2833 }
2834 return Register();
2835}
2836
2837Register fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
2838 if (RetVT.SimpleTy != MVT::f64)
2839 return Register();
2840 if ((Subtarget->hasFPARMv8())) {
2841 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMDr, RC: &AArch64::FPR64RegClass, Op0);
2842 }
2843 return Register();
2844}
2845
2846Register fastEmit_ISD_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
2847 if (RetVT.SimpleTy != MVT::v4f16)
2848 return Register();
2849 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2850 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f16, RC: &AArch64::FPR64RegClass, Op0);
2851 }
2852 return Register();
2853}
2854
2855Register fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
2856 if (RetVT.SimpleTy != MVT::v8f16)
2857 return Register();
2858 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2859 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv8f16, RC: &AArch64::FPR128RegClass, Op0);
2860 }
2861 return Register();
2862}
2863
2864Register fastEmit_ISD_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
2865 if (RetVT.SimpleTy != MVT::v2f32)
2866 return Register();
2867 if ((Subtarget->isNeonAvailable())) {
2868 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f32, RC: &AArch64::FPR64RegClass, Op0);
2869 }
2870 return Register();
2871}
2872
2873Register fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
2874 if (RetVT.SimpleTy != MVT::v4f32)
2875 return Register();
2876 if ((Subtarget->isNeonAvailable())) {
2877 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f32, RC: &AArch64::FPR128RegClass, Op0);
2878 }
2879 return Register();
2880}
2881
2882Register fastEmit_ISD_FFLOOR_MVT_v2f64_r(MVT RetVT, Register Op0) {
2883 if (RetVT.SimpleTy != MVT::v2f64)
2884 return Register();
2885 if ((Subtarget->isNeonAvailable())) {
2886 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f64, RC: &AArch64::FPR128RegClass, Op0);
2887 }
2888 return Register();
2889}
2890
2891Register fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
2892 switch (VT.SimpleTy) {
2893 case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0);
2894 case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0);
2895 case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0);
2896 case MVT::v4f16: return fastEmit_ISD_FFLOOR_MVT_v4f16_r(RetVT, Op0);
2897 case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0);
2898 case MVT::v2f32: return fastEmit_ISD_FFLOOR_MVT_v2f32_r(RetVT, Op0);
2899 case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0);
2900 case MVT::v2f64: return fastEmit_ISD_FFLOOR_MVT_v2f64_r(RetVT, Op0);
2901 default: return Register();
2902 }
2903}
2904
2905// FastEmit functions for ISD::FNEARBYINT.
2906
2907Register fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
2908 if (RetVT.SimpleTy != MVT::f16)
2909 return Register();
2910 if ((Subtarget->hasFullFP16())) {
2911 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIHr, RC: &AArch64::FPR16RegClass, Op0);
2912 }
2913 return Register();
2914}
2915
2916Register fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
2917 if (RetVT.SimpleTy != MVT::f32)
2918 return Register();
2919 if ((Subtarget->hasFPARMv8())) {
2920 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTISr, RC: &AArch64::FPR32RegClass, Op0);
2921 }
2922 return Register();
2923}
2924
2925Register fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
2926 if (RetVT.SimpleTy != MVT::f64)
2927 return Register();
2928 if ((Subtarget->hasFPARMv8())) {
2929 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIDr, RC: &AArch64::FPR64RegClass, Op0);
2930 }
2931 return Register();
2932}
2933
2934Register fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2935 if (RetVT.SimpleTy != MVT::v4f16)
2936 return Register();
2937 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2938 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f16, RC: &AArch64::FPR64RegClass, Op0);
2939 }
2940 return Register();
2941}
2942
2943Register fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2944 if (RetVT.SimpleTy != MVT::v8f16)
2945 return Register();
2946 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2947 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv8f16, RC: &AArch64::FPR128RegClass, Op0);
2948 }
2949 return Register();
2950}
2951
2952Register fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2953 if (RetVT.SimpleTy != MVT::v2f32)
2954 return Register();
2955 if ((Subtarget->isNeonAvailable())) {
2956 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f32, RC: &AArch64::FPR64RegClass, Op0);
2957 }
2958 return Register();
2959}
2960
2961Register fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2962 if (RetVT.SimpleTy != MVT::v4f32)
2963 return Register();
2964 if ((Subtarget->isNeonAvailable())) {
2965 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f32, RC: &AArch64::FPR128RegClass, Op0);
2966 }
2967 return Register();
2968}
2969
2970Register fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
2971 if (RetVT.SimpleTy != MVT::v2f64)
2972 return Register();
2973 if ((Subtarget->isNeonAvailable())) {
2974 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f64, RC: &AArch64::FPR128RegClass, Op0);
2975 }
2976 return Register();
2977}
2978
2979Register fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
2980 switch (VT.SimpleTy) {
2981 case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0);
2982 case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0);
2983 case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0);
2984 case MVT::v4f16: return fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(RetVT, Op0);
2985 case MVT::v8f16: return fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(RetVT, Op0);
2986 case MVT::v2f32: return fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(RetVT, Op0);
2987 case MVT::v4f32: return fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(RetVT, Op0);
2988 case MVT::v2f64: return fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(RetVT, Op0);
2989 default: return Register();
2990 }
2991}
2992
2993// FastEmit functions for ISD::FNEG.
2994
2995Register fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, Register Op0) {
2996 if (RetVT.SimpleTy != MVT::f16)
2997 return Register();
2998 if ((Subtarget->hasFullFP16())) {
2999 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGHr, RC: &AArch64::FPR16RegClass, Op0);
3000 }
3001 return Register();
3002}
3003
3004Register fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, Register Op0) {
3005 if (RetVT.SimpleTy != MVT::f32)
3006 return Register();
3007 if ((Subtarget->hasFPARMv8())) {
3008 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGSr, RC: &AArch64::FPR32RegClass, Op0);
3009 }
3010 return Register();
3011}
3012
3013Register fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, Register Op0) {
3014 if (RetVT.SimpleTy != MVT::f64)
3015 return Register();
3016 if ((Subtarget->hasFPARMv8())) {
3017 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGDr, RC: &AArch64::FPR64RegClass, Op0);
3018 }
3019 return Register();
3020}
3021
3022Register fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, Register Op0) {
3023 if (RetVT.SimpleTy != MVT::v4f16)
3024 return Register();
3025 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3026 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f16, RC: &AArch64::FPR64RegClass, Op0);
3027 }
3028 return Register();
3029}
3030
3031Register fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, Register Op0) {
3032 if (RetVT.SimpleTy != MVT::v8f16)
3033 return Register();
3034 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3035 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv8f16, RC: &AArch64::FPR128RegClass, Op0);
3036 }
3037 return Register();
3038}
3039
3040Register fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, Register Op0) {
3041 if (RetVT.SimpleTy != MVT::v2f32)
3042 return Register();
3043 if ((Subtarget->isNeonAvailable())) {
3044 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv2f32, RC: &AArch64::FPR64RegClass, Op0);
3045 }
3046 return Register();
3047}
3048
3049Register fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, Register Op0) {
3050 if (RetVT.SimpleTy != MVT::v4f32)
3051 return Register();
3052 if ((Subtarget->isNeonAvailable())) {
3053 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f32, RC: &AArch64::FPR128RegClass, Op0);
3054 }
3055 return Register();
3056}
3057
3058Register fastEmit_ISD_FNEG_MVT_v2f64_r(MVT RetVT, Register Op0) {
3059 if (RetVT.SimpleTy != MVT::v2f64)
3060 return Register();
3061 if ((Subtarget->isNeonAvailable())) {
3062 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv2f64, RC: &AArch64::FPR128RegClass, Op0);
3063 }
3064 return Register();
3065}
3066
3067Register fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, Register Op0) {
3068 switch (VT.SimpleTy) {
3069 case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0);
3070 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
3071 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
3072 case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0);
3073 case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0);
3074 case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0);
3075 case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0);
3076 case MVT::v2f64: return fastEmit_ISD_FNEG_MVT_v2f64_r(RetVT, Op0);
3077 default: return Register();
3078 }
3079}
3080
3081// FastEmit functions for ISD::FP_EXTEND.
3082
3083Register fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Register Op0) {
3084 if ((Subtarget->hasFPARMv8())) {
3085 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSHr, RC: &AArch64::FPR32RegClass, Op0);
3086 }
3087 return Register();
3088}
3089
3090Register fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Register Op0) {
3091 if ((Subtarget->hasFPARMv8())) {
3092 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDHr, RC: &AArch64::FPR64RegClass, Op0);
3093 }
3094 return Register();
3095}
3096
3097Register fastEmit_ISD_FP_EXTEND_MVT_f16_r(MVT RetVT, Register Op0) {
3098switch (RetVT.SimpleTy) {
3099 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Op0);
3100 case MVT::f64: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Op0);
3101 default: return Register();
3102}
3103}
3104
3105Register fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
3106 if (RetVT.SimpleTy != MVT::f64)
3107 return Register();
3108 if ((Subtarget->hasFPARMv8())) {
3109 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDSr, RC: &AArch64::FPR64RegClass, Op0);
3110 }
3111 return Register();
3112}
3113
3114Register fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3115 if (RetVT.SimpleTy != MVT::v4f32)
3116 return Register();
3117 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv4i16, RC: &AArch64::FPR128RegClass, Op0);
3118}
3119
3120Register fastEmit_ISD_FP_EXTEND_MVT_v4bf16_r(MVT RetVT, Register Op0) {
3121 if (RetVT.SimpleTy != MVT::v4f32)
3122 return Register();
3123 if ((Subtarget->isNeonAvailable())) {
3124 return fastEmitInst_r(MachineInstOpcode: AArch64::SHLLv4i16, RC: &AArch64::FPR128RegClass, Op0);
3125 }
3126 return Register();
3127}
3128
3129Register fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3130 if (RetVT.SimpleTy != MVT::v2f64)
3131 return Register();
3132 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv2i32, RC: &AArch64::FPR128RegClass, Op0);
3133}
3134
3135Register fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3136 switch (VT.SimpleTy) {
3137 case MVT::f16: return fastEmit_ISD_FP_EXTEND_MVT_f16_r(RetVT, Op0);
3138 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
3139 case MVT::v4f16: return fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
3140 case MVT::v4bf16: return fastEmit_ISD_FP_EXTEND_MVT_v4bf16_r(RetVT, Op0);
3141 case MVT::v2f32: return fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(RetVT, Op0);
3142 default: return Register();
3143 }
3144}
3145
3146// FastEmit functions for ISD::FP_ROUND.
3147
3148Register fastEmit_ISD_FP_ROUND_MVT_f32_MVT_bf16_r(Register Op0) {
3149 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
3150 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVT, RC: &AArch64::FPR16RegClass, Op0);
3151 }
3152 return Register();
3153}
3154
3155Register fastEmit_ISD_FP_ROUND_MVT_f32_MVT_f16_r(Register Op0) {
3156 if ((Subtarget->hasFPARMv8())) {
3157 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHSr, RC: &AArch64::FPR16RegClass, Op0);
3158 }
3159 return Register();
3160}
3161
3162Register fastEmit_ISD_FP_ROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3163switch (RetVT.SimpleTy) {
3164 case MVT::bf16: return fastEmit_ISD_FP_ROUND_MVT_f32_MVT_bf16_r(Op0);
3165 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f32_MVT_f16_r(Op0);
3166 default: return Register();
3167}
3168}
3169
3170Register fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Register Op0) {
3171 if ((Subtarget->hasFPARMv8())) {
3172 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHDr, RC: &AArch64::FPR16RegClass, Op0);
3173 }
3174 return Register();
3175}
3176
3177Register fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Register Op0) {
3178 if ((Subtarget->hasFPARMv8())) {
3179 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSDr, RC: &AArch64::FPR32RegClass, Op0);
3180 }
3181 return Register();
3182}
3183
3184Register fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3185switch (RetVT.SimpleTy) {
3186 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Op0);
3187 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Op0);
3188 default: return Register();
3189}
3190}
3191
3192Register fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Register Op0) {
3193 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
3194}
3195
3196Register fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Register Op0) {
3197 if ((Subtarget->hasBF16()) && (Subtarget->isNeonAvailable())) {
3198 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVTN, RC: &AArch64::FPR64RegClass, Op0);
3199 }
3200 return Register();
3201}
3202
3203Register fastEmit_ISD_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3204switch (RetVT.SimpleTy) {
3205 case MVT::v4f16: return fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Op0);
3206 case MVT::v4bf16: return fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Op0);
3207 default: return Register();
3208}
3209}
3210
3211Register fastEmit_ISD_FP_ROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
3212 if (RetVT.SimpleTy != MVT::v2f32)
3213 return Register();
3214 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
3215}
3216
3217Register fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
3218 switch (VT.SimpleTy) {
3219 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f32_r(RetVT, Op0);
3220 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
3221 case MVT::v4f32: return fastEmit_ISD_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
3222 case MVT::v2f64: return fastEmit_ISD_FP_ROUND_MVT_v2f64_r(RetVT, Op0);
3223 default: return Register();
3224 }
3225}
3226
3227// FastEmit functions for ISD::FP_TO_SINT.
3228
3229Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
3230 if ((Subtarget->hasFPRCVT())) {
3231 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSHr, RC: &AArch64::FPR32RegClass, Op0);
3232 }
3233 if ((Subtarget->hasFullFP16())) {
3234 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
3235 }
3236 return Register();
3237}
3238
3239Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
3240 if ((Subtarget->hasFPRCVT())) {
3241 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDHr, RC: &AArch64::FPR64RegClass, Op0);
3242 }
3243 if ((Subtarget->hasFullFP16())) {
3244 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
3245 }
3246 return Register();
3247}
3248
3249Register fastEmit_ISD_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
3250switch (RetVT.SimpleTy) {
3251 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
3252 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
3253 default: return Register();
3254}
3255}
3256
3257Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
3258 if ((Subtarget->hasFPARMv8())) {
3259 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
3260 }
3261 return Register();
3262}
3263
3264Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
3265 if ((Subtarget->hasFPRCVT())) {
3266 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDSr, RC: &AArch64::FPR64RegClass, Op0);
3267 }
3268 if ((Subtarget->hasFPARMv8())) {
3269 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
3270 }
3271 return Register();
3272}
3273
3274Register fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
3275switch (RetVT.SimpleTy) {
3276 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
3277 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
3278 default: return Register();
3279}
3280}
3281
3282Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
3283 if ((Subtarget->hasFPRCVT())) {
3284 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSDr, RC: &AArch64::FPR32RegClass, Op0);
3285 }
3286 if ((Subtarget->hasFPARMv8())) {
3287 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
3288 }
3289 return Register();
3290}
3291
3292Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
3293 if ((Subtarget->hasFPARMv8())) {
3294 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
3295 }
3296 return Register();
3297}
3298
3299Register fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
3300switch (RetVT.SimpleTy) {
3301 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
3302 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
3303 default: return Register();
3304}
3305}
3306
3307Register fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3308 if (RetVT.SimpleTy != MVT::v4i16)
3309 return Register();
3310 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3311 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
3312 }
3313 return Register();
3314}
3315
3316Register fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3317 if (RetVT.SimpleTy != MVT::v8i16)
3318 return Register();
3319 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3320 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
3321 }
3322 return Register();
3323}
3324
3325Register fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3326 if (RetVT.SimpleTy != MVT::v2i32)
3327 return Register();
3328 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3329 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
3330 }
3331 return Register();
3332}
3333
3334Register fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3335 if (RetVT.SimpleTy != MVT::v4i32)
3336 return Register();
3337 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3338 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
3339 }
3340 return Register();
3341}
3342
3343Register fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3344 if (RetVT.SimpleTy != MVT::v2i64)
3345 return Register();
3346 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3347 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
3348 }
3349 return Register();
3350}
3351
3352Register fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
3353 switch (VT.SimpleTy) {
3354 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
3355 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
3356 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
3357 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
3358 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
3359 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
3360 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
3361 case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0);
3362 default: return Register();
3363 }
3364}
3365
3366// FastEmit functions for ISD::FP_TO_SINT_SAT.
3367
3368Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_MVT_i32_r(Register Op0) {
3369 if ((Subtarget->hasFPRCVT())) {
3370 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSHr, RC: &AArch64::FPR32RegClass, Op0);
3371 }
3372 if ((Subtarget->hasFullFP16())) {
3373 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
3374 }
3375 return Register();
3376}
3377
3378Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_MVT_i64_r(Register Op0) {
3379 if ((Subtarget->hasFPRCVT())) {
3380 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDHr, RC: &AArch64::FPR64RegClass, Op0);
3381 }
3382 if ((Subtarget->hasFullFP16())) {
3383 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
3384 }
3385 return Register();
3386}
3387
3388Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_r(MVT RetVT, Register Op0) {
3389switch (RetVT.SimpleTy) {
3390 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_MVT_i32_r(Op0);
3391 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_MVT_i64_r(Op0);
3392 default: return Register();
3393}
3394}
3395
3396Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_MVT_i32_r(Register Op0) {
3397 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
3398}
3399
3400Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_MVT_i64_r(Register Op0) {
3401 if ((Subtarget->hasFPRCVT())) {
3402 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDSr, RC: &AArch64::FPR64RegClass, Op0);
3403 }
3404 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
3405}
3406
3407Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_r(MVT RetVT, Register Op0) {
3408switch (RetVT.SimpleTy) {
3409 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_MVT_i32_r(Op0);
3410 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_MVT_i64_r(Op0);
3411 default: return Register();
3412}
3413}
3414
3415Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_MVT_i32_r(Register Op0) {
3416 if ((Subtarget->hasFPRCVT())) {
3417 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSDr, RC: &AArch64::FPR32RegClass, Op0);
3418 }
3419 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
3420}
3421
3422Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_MVT_i64_r(Register Op0) {
3423 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
3424}
3425
3426Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_r(MVT RetVT, Register Op0) {
3427switch (RetVT.SimpleTy) {
3428 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_MVT_i32_r(Op0);
3429 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_MVT_i64_r(Op0);
3430 default: return Register();
3431}
3432}
3433
3434Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3435 if (RetVT.SimpleTy != MVT::v4i16)
3436 return Register();
3437 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3438 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
3439 }
3440 return Register();
3441}
3442
3443Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3444 if (RetVT.SimpleTy != MVT::v8i16)
3445 return Register();
3446 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3447 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
3448 }
3449 return Register();
3450}
3451
3452Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3453 if (RetVT.SimpleTy != MVT::v2i32)
3454 return Register();
3455 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3456 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
3457 }
3458 return Register();
3459}
3460
3461Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3462 if (RetVT.SimpleTy != MVT::v4i32)
3463 return Register();
3464 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3465 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
3466 }
3467 return Register();
3468}
3469
3470Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3471 if (RetVT.SimpleTy != MVT::v2i64)
3472 return Register();
3473 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3474 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
3475 }
3476 return Register();
3477}
3478
3479Register fastEmit_ISD_FP_TO_SINT_SAT_r(MVT VT, MVT RetVT, Register Op0) {
3480 switch (VT.SimpleTy) {
3481 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_r(RetVT, Op0);
3482 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_r(RetVT, Op0);
3483 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_r(RetVT, Op0);
3484 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_v4f16_r(RetVT, Op0);
3485 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_v8f16_r(RetVT, Op0);
3486 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_v2f32_r(RetVT, Op0);
3487 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_v4f32_r(RetVT, Op0);
3488 case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_v2f64_r(RetVT, Op0);
3489 default: return Register();
3490 }
3491}
3492
3493// FastEmit functions for ISD::FP_TO_UINT.
3494
3495Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
3496 if ((Subtarget->hasFPRCVT())) {
3497 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSHr, RC: &AArch64::FPR32RegClass, Op0);
3498 }
3499 if ((Subtarget->hasFullFP16())) {
3500 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
3501 }
3502 return Register();
3503}
3504
3505Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
3506 if ((Subtarget->hasFPRCVT())) {
3507 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDHr, RC: &AArch64::FPR64RegClass, Op0);
3508 }
3509 if ((Subtarget->hasFullFP16())) {
3510 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
3511 }
3512 return Register();
3513}
3514
3515Register fastEmit_ISD_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
3516switch (RetVT.SimpleTy) {
3517 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
3518 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
3519 default: return Register();
3520}
3521}
3522
3523Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
3524 if ((Subtarget->hasFPARMv8())) {
3525 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
3526 }
3527 return Register();
3528}
3529
3530Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
3531 if ((Subtarget->hasFPRCVT())) {
3532 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDSr, RC: &AArch64::FPR64RegClass, Op0);
3533 }
3534 if ((Subtarget->hasFPARMv8())) {
3535 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
3536 }
3537 return Register();
3538}
3539
3540Register fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
3541switch (RetVT.SimpleTy) {
3542 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
3543 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
3544 default: return Register();
3545}
3546}
3547
3548Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
3549 if ((Subtarget->hasFPRCVT())) {
3550 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSDr, RC: &AArch64::FPR32RegClass, Op0);
3551 }
3552 if ((Subtarget->hasFPARMv8())) {
3553 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
3554 }
3555 return Register();
3556}
3557
3558Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
3559 if ((Subtarget->hasFPARMv8())) {
3560 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
3561 }
3562 return Register();
3563}
3564
3565Register fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
3566switch (RetVT.SimpleTy) {
3567 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
3568 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
3569 default: return Register();
3570}
3571}
3572
3573Register fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3574 if (RetVT.SimpleTy != MVT::v4i16)
3575 return Register();
3576 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3577 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
3578 }
3579 return Register();
3580}
3581
3582Register fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3583 if (RetVT.SimpleTy != MVT::v8i16)
3584 return Register();
3585 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3586 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
3587 }
3588 return Register();
3589}
3590
3591Register fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3592 if (RetVT.SimpleTy != MVT::v2i32)
3593 return Register();
3594 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3595 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
3596 }
3597 return Register();
3598}
3599
3600Register fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3601 if (RetVT.SimpleTy != MVT::v4i32)
3602 return Register();
3603 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3604 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
3605 }
3606 return Register();
3607}
3608
3609Register fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3610 if (RetVT.SimpleTy != MVT::v2i64)
3611 return Register();
3612 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3613 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
3614 }
3615 return Register();
3616}
3617
3618Register fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
3619 switch (VT.SimpleTy) {
3620 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
3621 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
3622 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
3623 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
3624 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
3625 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
3626 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
3627 case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0);
3628 default: return Register();
3629 }
3630}
3631
3632// FastEmit functions for ISD::FP_TO_UINT_SAT.
3633
3634Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_MVT_i32_r(Register Op0) {
3635 if ((Subtarget->hasFPRCVT())) {
3636 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSHr, RC: &AArch64::FPR32RegClass, Op0);
3637 }
3638 if ((Subtarget->hasFullFP16())) {
3639 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
3640 }
3641 return Register();
3642}
3643
3644Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_MVT_i64_r(Register Op0) {
3645 if ((Subtarget->hasFPRCVT())) {
3646 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDHr, RC: &AArch64::FPR64RegClass, Op0);
3647 }
3648 if ((Subtarget->hasFullFP16())) {
3649 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
3650 }
3651 return Register();
3652}
3653
3654Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_r(MVT RetVT, Register Op0) {
3655switch (RetVT.SimpleTy) {
3656 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_MVT_i32_r(Op0);
3657 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_MVT_i64_r(Op0);
3658 default: return Register();
3659}
3660}
3661
3662Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_MVT_i32_r(Register Op0) {
3663 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
3664}
3665
3666Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_MVT_i64_r(Register Op0) {
3667 if ((Subtarget->hasFPRCVT())) {
3668 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDSr, RC: &AArch64::FPR64RegClass, Op0);
3669 }
3670 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
3671}
3672
3673Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_r(MVT RetVT, Register Op0) {
3674switch (RetVT.SimpleTy) {
3675 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_MVT_i32_r(Op0);
3676 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_MVT_i64_r(Op0);
3677 default: return Register();
3678}
3679}
3680
3681Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_MVT_i32_r(Register Op0) {
3682 if ((Subtarget->hasFPRCVT())) {
3683 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSDr, RC: &AArch64::FPR32RegClass, Op0);
3684 }
3685 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
3686}
3687
3688Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_MVT_i64_r(Register Op0) {
3689 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
3690}
3691
3692Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_r(MVT RetVT, Register Op0) {
3693switch (RetVT.SimpleTy) {
3694 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_MVT_i32_r(Op0);
3695 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_MVT_i64_r(Op0);
3696 default: return Register();
3697}
3698}
3699
3700Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3701 if (RetVT.SimpleTy != MVT::v4i16)
3702 return Register();
3703 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3704 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
3705 }
3706 return Register();
3707}
3708
3709Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3710 if (RetVT.SimpleTy != MVT::v8i16)
3711 return Register();
3712 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3713 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
3714 }
3715 return Register();
3716}
3717
3718Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3719 if (RetVT.SimpleTy != MVT::v2i32)
3720 return Register();
3721 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3722 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
3723 }
3724 return Register();
3725}
3726
3727Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3728 if (RetVT.SimpleTy != MVT::v4i32)
3729 return Register();
3730 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3731 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
3732 }
3733 return Register();
3734}
3735
3736Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3737 if (RetVT.SimpleTy != MVT::v2i64)
3738 return Register();
3739 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3740 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
3741 }
3742 return Register();
3743}
3744
3745Register fastEmit_ISD_FP_TO_UINT_SAT_r(MVT VT, MVT RetVT, Register Op0) {
3746 switch (VT.SimpleTy) {
3747 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_r(RetVT, Op0);
3748 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_r(RetVT, Op0);
3749 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_r(RetVT, Op0);
3750 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_v4f16_r(RetVT, Op0);
3751 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_v8f16_r(RetVT, Op0);
3752 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_v2f32_r(RetVT, Op0);
3753 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_v4f32_r(RetVT, Op0);
3754 case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_v2f64_r(RetVT, Op0);
3755 default: return Register();
3756 }
3757}
3758
3759// FastEmit functions for ISD::FRINT.
3760
3761Register fastEmit_ISD_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
3762 if (RetVT.SimpleTy != MVT::f16)
3763 return Register();
3764 if ((Subtarget->hasFullFP16())) {
3765 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXHr, RC: &AArch64::FPR16RegClass, Op0);
3766 }
3767 return Register();
3768}
3769
3770Register fastEmit_ISD_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
3771 if (RetVT.SimpleTy != MVT::f32)
3772 return Register();
3773 if ((Subtarget->hasFPARMv8())) {
3774 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXSr, RC: &AArch64::FPR32RegClass, Op0);
3775 }
3776 return Register();
3777}
3778
3779Register fastEmit_ISD_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
3780 if (RetVT.SimpleTy != MVT::f64)
3781 return Register();
3782 if ((Subtarget->hasFPARMv8())) {
3783 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXDr, RC: &AArch64::FPR64RegClass, Op0);
3784 }
3785 return Register();
3786}
3787
3788Register fastEmit_ISD_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3789 if (RetVT.SimpleTy != MVT::v4f16)
3790 return Register();
3791 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3792 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f16, RC: &AArch64::FPR64RegClass, Op0);
3793 }
3794 return Register();
3795}
3796
3797Register fastEmit_ISD_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3798 if (RetVT.SimpleTy != MVT::v8f16)
3799 return Register();
3800 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3801 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv8f16, RC: &AArch64::FPR128RegClass, Op0);
3802 }
3803 return Register();
3804}
3805
3806Register fastEmit_ISD_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3807 if (RetVT.SimpleTy != MVT::v2f32)
3808 return Register();
3809 if ((Subtarget->isNeonAvailable())) {
3810 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f32, RC: &AArch64::FPR64RegClass, Op0);
3811 }
3812 return Register();
3813}
3814
3815Register fastEmit_ISD_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3816 if (RetVT.SimpleTy != MVT::v4f32)
3817 return Register();
3818 if ((Subtarget->isNeonAvailable())) {
3819 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f32, RC: &AArch64::FPR128RegClass, Op0);
3820 }
3821 return Register();
3822}
3823
3824Register fastEmit_ISD_FRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3825 if (RetVT.SimpleTy != MVT::v2f64)
3826 return Register();
3827 if ((Subtarget->isNeonAvailable())) {
3828 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f64, RC: &AArch64::FPR128RegClass, Op0);
3829 }
3830 return Register();
3831}
3832
3833Register fastEmit_ISD_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
3834 switch (VT.SimpleTy) {
3835 case MVT::f16: return fastEmit_ISD_FRINT_MVT_f16_r(RetVT, Op0);
3836 case MVT::f32: return fastEmit_ISD_FRINT_MVT_f32_r(RetVT, Op0);
3837 case MVT::f64: return fastEmit_ISD_FRINT_MVT_f64_r(RetVT, Op0);
3838 case MVT::v4f16: return fastEmit_ISD_FRINT_MVT_v4f16_r(RetVT, Op0);
3839 case MVT::v8f16: return fastEmit_ISD_FRINT_MVT_v8f16_r(RetVT, Op0);
3840 case MVT::v2f32: return fastEmit_ISD_FRINT_MVT_v2f32_r(RetVT, Op0);
3841 case MVT::v4f32: return fastEmit_ISD_FRINT_MVT_v4f32_r(RetVT, Op0);
3842 case MVT::v2f64: return fastEmit_ISD_FRINT_MVT_v2f64_r(RetVT, Op0);
3843 default: return Register();
3844 }
3845}
3846
3847// FastEmit functions for ISD::FROUND.
3848
3849Register fastEmit_ISD_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
3850 if (RetVT.SimpleTy != MVT::f16)
3851 return Register();
3852 if ((Subtarget->hasFullFP16())) {
3853 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAHr, RC: &AArch64::FPR16RegClass, Op0);
3854 }
3855 return Register();
3856}
3857
3858Register fastEmit_ISD_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3859 if (RetVT.SimpleTy != MVT::f32)
3860 return Register();
3861 if ((Subtarget->hasFPARMv8())) {
3862 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTASr, RC: &AArch64::FPR32RegClass, Op0);
3863 }
3864 return Register();
3865}
3866
3867Register fastEmit_ISD_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3868 if (RetVT.SimpleTy != MVT::f64)
3869 return Register();
3870 if ((Subtarget->hasFPARMv8())) {
3871 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTADr, RC: &AArch64::FPR64RegClass, Op0);
3872 }
3873 return Register();
3874}
3875
3876Register fastEmit_ISD_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3877 if (RetVT.SimpleTy != MVT::v4f16)
3878 return Register();
3879 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3880 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f16, RC: &AArch64::FPR64RegClass, Op0);
3881 }
3882 return Register();
3883}
3884
3885Register fastEmit_ISD_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
3886 if (RetVT.SimpleTy != MVT::v8f16)
3887 return Register();
3888 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3889 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv8f16, RC: &AArch64::FPR128RegClass, Op0);
3890 }
3891 return Register();
3892}
3893
3894Register fastEmit_ISD_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3895 if (RetVT.SimpleTy != MVT::v2f32)
3896 return Register();
3897 if ((Subtarget->isNeonAvailable())) {
3898 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f32, RC: &AArch64::FPR64RegClass, Op0);
3899 }
3900 return Register();
3901}
3902
3903Register fastEmit_ISD_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3904 if (RetVT.SimpleTy != MVT::v4f32)
3905 return Register();
3906 if ((Subtarget->isNeonAvailable())) {
3907 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f32, RC: &AArch64::FPR128RegClass, Op0);
3908 }
3909 return Register();
3910}
3911
3912Register fastEmit_ISD_FROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
3913 if (RetVT.SimpleTy != MVT::v2f64)
3914 return Register();
3915 if ((Subtarget->isNeonAvailable())) {
3916 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f64, RC: &AArch64::FPR128RegClass, Op0);
3917 }
3918 return Register();
3919}
3920
3921Register fastEmit_ISD_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
3922 switch (VT.SimpleTy) {
3923 case MVT::f16: return fastEmit_ISD_FROUND_MVT_f16_r(RetVT, Op0);
3924 case MVT::f32: return fastEmit_ISD_FROUND_MVT_f32_r(RetVT, Op0);
3925 case MVT::f64: return fastEmit_ISD_FROUND_MVT_f64_r(RetVT, Op0);
3926 case MVT::v4f16: return fastEmit_ISD_FROUND_MVT_v4f16_r(RetVT, Op0);
3927 case MVT::v8f16: return fastEmit_ISD_FROUND_MVT_v8f16_r(RetVT, Op0);
3928 case MVT::v2f32: return fastEmit_ISD_FROUND_MVT_v2f32_r(RetVT, Op0);
3929 case MVT::v4f32: return fastEmit_ISD_FROUND_MVT_v4f32_r(RetVT, Op0);
3930 case MVT::v2f64: return fastEmit_ISD_FROUND_MVT_v2f64_r(RetVT, Op0);
3931 default: return Register();
3932 }
3933}
3934
3935// FastEmit functions for ISD::FROUNDEVEN.
3936
3937Register fastEmit_ISD_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
3938 if (RetVT.SimpleTy != MVT::f16)
3939 return Register();
3940 if ((Subtarget->hasFullFP16())) {
3941 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNHr, RC: &AArch64::FPR16RegClass, Op0);
3942 }
3943 return Register();
3944}
3945
3946Register fastEmit_ISD_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
3947 if (RetVT.SimpleTy != MVT::f32)
3948 return Register();
3949 if ((Subtarget->hasFPARMv8())) {
3950 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNSr, RC: &AArch64::FPR32RegClass, Op0);
3951 }
3952 return Register();
3953}
3954
3955Register fastEmit_ISD_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
3956 if (RetVT.SimpleTy != MVT::f64)
3957 return Register();
3958 if ((Subtarget->hasFPARMv8())) {
3959 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNDr, RC: &AArch64::FPR64RegClass, Op0);
3960 }
3961 return Register();
3962}
3963
3964Register fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
3965 if (RetVT.SimpleTy != MVT::v4f16)
3966 return Register();
3967 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3968 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f16, RC: &AArch64::FPR64RegClass, Op0);
3969 }
3970 return Register();
3971}
3972
3973Register fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
3974 if (RetVT.SimpleTy != MVT::v8f16)
3975 return Register();
3976 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3977 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv8f16, RC: &AArch64::FPR128RegClass, Op0);
3978 }
3979 return Register();
3980}
3981
3982Register fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
3983 if (RetVT.SimpleTy != MVT::v2f32)
3984 return Register();
3985 if ((Subtarget->isNeonAvailable())) {
3986 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f32, RC: &AArch64::FPR64RegClass, Op0);
3987 }
3988 return Register();
3989}
3990
3991Register fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
3992 if (RetVT.SimpleTy != MVT::v4f32)
3993 return Register();
3994 if ((Subtarget->isNeonAvailable())) {
3995 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f32, RC: &AArch64::FPR128RegClass, Op0);
3996 }
3997 return Register();
3998}
3999
4000Register fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, Register Op0) {
4001 if (RetVT.SimpleTy != MVT::v2f64)
4002 return Register();
4003 if ((Subtarget->isNeonAvailable())) {
4004 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f64, RC: &AArch64::FPR128RegClass, Op0);
4005 }
4006 return Register();
4007}
4008
4009Register fastEmit_ISD_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
4010 switch (VT.SimpleTy) {
4011 case MVT::f16: return fastEmit_ISD_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
4012 case MVT::f32: return fastEmit_ISD_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
4013 case MVT::f64: return fastEmit_ISD_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
4014 case MVT::v4f16: return fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
4015 case MVT::v8f16: return fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
4016 case MVT::v2f32: return fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
4017 case MVT::v4f32: return fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
4018 case MVT::v2f64: return fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0);
4019 default: return Register();
4020 }
4021}
4022
4023// FastEmit functions for ISD::FSQRT.
4024
4025Register fastEmit_ISD_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
4026 if (RetVT.SimpleTy != MVT::f16)
4027 return Register();
4028 if ((Subtarget->hasFullFP16())) {
4029 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTHr, RC: &AArch64::FPR16RegClass, Op0);
4030 }
4031 return Register();
4032}
4033
4034Register fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
4035 if (RetVT.SimpleTy != MVT::f32)
4036 return Register();
4037 if ((Subtarget->hasFPARMv8())) {
4038 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTSr, RC: &AArch64::FPR32RegClass, Op0);
4039 }
4040 return Register();
4041}
4042
4043Register fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
4044 if (RetVT.SimpleTy != MVT::f64)
4045 return Register();
4046 if ((Subtarget->hasFPARMv8())) {
4047 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTDr, RC: &AArch64::FPR64RegClass, Op0);
4048 }
4049 return Register();
4050}
4051
4052Register fastEmit_ISD_FSQRT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4053 if (RetVT.SimpleTy != MVT::v4f16)
4054 return Register();
4055 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4056 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f16, RC: &AArch64::FPR64RegClass, Op0);
4057 }
4058 return Register();
4059}
4060
4061Register fastEmit_ISD_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4062 if (RetVT.SimpleTy != MVT::v8f16)
4063 return Register();
4064 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4065 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv8f16, RC: &AArch64::FPR128RegClass, Op0);
4066 }
4067 return Register();
4068}
4069
4070Register fastEmit_ISD_FSQRT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4071 if (RetVT.SimpleTy != MVT::v2f32)
4072 return Register();
4073 if ((Subtarget->isNeonAvailable())) {
4074 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f32, RC: &AArch64::FPR64RegClass, Op0);
4075 }
4076 return Register();
4077}
4078
4079Register fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4080 if (RetVT.SimpleTy != MVT::v4f32)
4081 return Register();
4082 if ((Subtarget->isNeonAvailable())) {
4083 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f32, RC: &AArch64::FPR128RegClass, Op0);
4084 }
4085 return Register();
4086}
4087
4088Register fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4089 if (RetVT.SimpleTy != MVT::v2f64)
4090 return Register();
4091 if ((Subtarget->isNeonAvailable())) {
4092 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f64, RC: &AArch64::FPR128RegClass, Op0);
4093 }
4094 return Register();
4095}
4096
4097Register fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
4098 switch (VT.SimpleTy) {
4099 case MVT::f16: return fastEmit_ISD_FSQRT_MVT_f16_r(RetVT, Op0);
4100 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
4101 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
4102 case MVT::v4f16: return fastEmit_ISD_FSQRT_MVT_v4f16_r(RetVT, Op0);
4103 case MVT::v8f16: return fastEmit_ISD_FSQRT_MVT_v8f16_r(RetVT, Op0);
4104 case MVT::v2f32: return fastEmit_ISD_FSQRT_MVT_v2f32_r(RetVT, Op0);
4105 case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0);
4106 case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0);
4107 default: return Register();
4108 }
4109}
4110
4111// FastEmit functions for ISD::FTRUNC.
4112
4113Register fastEmit_ISD_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
4114 if (RetVT.SimpleTy != MVT::f16)
4115 return Register();
4116 if ((Subtarget->hasFullFP16())) {
4117 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZHr, RC: &AArch64::FPR16RegClass, Op0);
4118 }
4119 return Register();
4120}
4121
4122Register fastEmit_ISD_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
4123 if (RetVT.SimpleTy != MVT::f32)
4124 return Register();
4125 if ((Subtarget->hasFPARMv8())) {
4126 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZSr, RC: &AArch64::FPR32RegClass, Op0);
4127 }
4128 return Register();
4129}
4130
4131Register fastEmit_ISD_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
4132 if (RetVT.SimpleTy != MVT::f64)
4133 return Register();
4134 if ((Subtarget->hasFPARMv8())) {
4135 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZDr, RC: &AArch64::FPR64RegClass, Op0);
4136 }
4137 return Register();
4138}
4139
4140Register fastEmit_ISD_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
4141 if (RetVT.SimpleTy != MVT::v4f16)
4142 return Register();
4143 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4144 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f16, RC: &AArch64::FPR64RegClass, Op0);
4145 }
4146 return Register();
4147}
4148
4149Register fastEmit_ISD_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
4150 if (RetVT.SimpleTy != MVT::v8f16)
4151 return Register();
4152 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4153 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv8f16, RC: &AArch64::FPR128RegClass, Op0);
4154 }
4155 return Register();
4156}
4157
4158Register fastEmit_ISD_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
4159 if (RetVT.SimpleTy != MVT::v2f32)
4160 return Register();
4161 if ((Subtarget->isNeonAvailable())) {
4162 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f32, RC: &AArch64::FPR64RegClass, Op0);
4163 }
4164 return Register();
4165}
4166
4167Register fastEmit_ISD_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
4168 if (RetVT.SimpleTy != MVT::v4f32)
4169 return Register();
4170 if ((Subtarget->isNeonAvailable())) {
4171 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f32, RC: &AArch64::FPR128RegClass, Op0);
4172 }
4173 return Register();
4174}
4175
4176Register fastEmit_ISD_FTRUNC_MVT_v2f64_r(MVT RetVT, Register Op0) {
4177 if (RetVT.SimpleTy != MVT::v2f64)
4178 return Register();
4179 if ((Subtarget->isNeonAvailable())) {
4180 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f64, RC: &AArch64::FPR128RegClass, Op0);
4181 }
4182 return Register();
4183}
4184
4185Register fastEmit_ISD_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
4186 switch (VT.SimpleTy) {
4187 case MVT::f16: return fastEmit_ISD_FTRUNC_MVT_f16_r(RetVT, Op0);
4188 case MVT::f32: return fastEmit_ISD_FTRUNC_MVT_f32_r(RetVT, Op0);
4189 case MVT::f64: return fastEmit_ISD_FTRUNC_MVT_f64_r(RetVT, Op0);
4190 case MVT::v4f16: return fastEmit_ISD_FTRUNC_MVT_v4f16_r(RetVT, Op0);
4191 case MVT::v8f16: return fastEmit_ISD_FTRUNC_MVT_v8f16_r(RetVT, Op0);
4192 case MVT::v2f32: return fastEmit_ISD_FTRUNC_MVT_v2f32_r(RetVT, Op0);
4193 case MVT::v4f32: return fastEmit_ISD_FTRUNC_MVT_v4f32_r(RetVT, Op0);
4194 case MVT::v2f64: return fastEmit_ISD_FTRUNC_MVT_v2f64_r(RetVT, Op0);
4195 default: return Register();
4196 }
4197}
4198
4199// FastEmit functions for ISD::LLROUND.
4200
4201Register fastEmit_ISD_LLROUND_MVT_f16_r(MVT RetVT, Register Op0) {
4202 if (RetVT.SimpleTy != MVT::i64)
4203 return Register();
4204 if ((Subtarget->hasFPRCVT())) {
4205 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDHr, RC: &AArch64::FPR64RegClass, Op0);
4206 }
4207 if ((Subtarget->hasFullFP16())) {
4208 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
4209 }
4210 return Register();
4211}
4212
4213Register fastEmit_ISD_LLROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4214 if (RetVT.SimpleTy != MVT::i64)
4215 return Register();
4216 if ((Subtarget->hasFPRCVT())) {
4217 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDSr, RC: &AArch64::FPR64RegClass, Op0);
4218 }
4219 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
4220}
4221
4222Register fastEmit_ISD_LLROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4223 if (RetVT.SimpleTy != MVT::i64)
4224 return Register();
4225 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
4226}
4227
4228Register fastEmit_ISD_LLROUND_r(MVT VT, MVT RetVT, Register Op0) {
4229 switch (VT.SimpleTy) {
4230 case MVT::f16: return fastEmit_ISD_LLROUND_MVT_f16_r(RetVT, Op0);
4231 case MVT::f32: return fastEmit_ISD_LLROUND_MVT_f32_r(RetVT, Op0);
4232 case MVT::f64: return fastEmit_ISD_LLROUND_MVT_f64_r(RetVT, Op0);
4233 default: return Register();
4234 }
4235}
4236
4237// FastEmit functions for ISD::LROUND.
4238
4239Register fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(Register Op0) {
4240 if ((Subtarget->hasFPRCVT())) {
4241 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASSHr, RC: &AArch64::FPR32RegClass, Op0);
4242 }
4243 if ((Subtarget->hasFullFP16())) {
4244 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWHr, RC: &AArch64::GPR32RegClass, Op0);
4245 }
4246 return Register();
4247}
4248
4249Register fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(Register Op0) {
4250 if ((Subtarget->hasFPRCVT())) {
4251 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDHr, RC: &AArch64::FPR64RegClass, Op0);
4252 }
4253 if ((Subtarget->hasFullFP16())) {
4254 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
4255 }
4256 return Register();
4257}
4258
4259Register fastEmit_ISD_LROUND_MVT_f16_r(MVT RetVT, Register Op0) {
4260switch (RetVT.SimpleTy) {
4261 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(Op0);
4262 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(Op0);
4263 default: return Register();
4264}
4265}
4266
4267Register fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(Register Op0) {
4268 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWSr, RC: &AArch64::GPR32RegClass, Op0);
4269}
4270
4271Register fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(Register Op0) {
4272 if ((Subtarget->hasFPRCVT())) {
4273 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDSr, RC: &AArch64::FPR64RegClass, Op0);
4274 }
4275 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
4276}
4277
4278Register fastEmit_ISD_LROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4279switch (RetVT.SimpleTy) {
4280 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(Op0);
4281 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(Op0);
4282 default: return Register();
4283}
4284}
4285
4286Register fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(Register Op0) {
4287 if ((Subtarget->hasFPRCVT())) {
4288 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASSDr, RC: &AArch64::FPR32RegClass, Op0);
4289 }
4290 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWDr, RC: &AArch64::GPR32RegClass, Op0);
4291}
4292
4293Register fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(Register Op0) {
4294 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
4295}
4296
4297Register fastEmit_ISD_LROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4298switch (RetVT.SimpleTy) {
4299 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(Op0);
4300 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(Op0);
4301 default: return Register();
4302}
4303}
4304
4305Register fastEmit_ISD_LROUND_r(MVT VT, MVT RetVT, Register Op0) {
4306 switch (VT.SimpleTy) {
4307 case MVT::f16: return fastEmit_ISD_LROUND_MVT_f16_r(RetVT, Op0);
4308 case MVT::f32: return fastEmit_ISD_LROUND_MVT_f32_r(RetVT, Op0);
4309 case MVT::f64: return fastEmit_ISD_LROUND_MVT_f64_r(RetVT, Op0);
4310 default: return Register();
4311 }
4312}
4313
4314// FastEmit functions for ISD::SINT_TO_FP.
4315
4316Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
4317 if ((Subtarget->hasFPRCVT())) {
4318 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
4319 }
4320 if ((Subtarget->hasFullFP16())) {
4321 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
4322 }
4323 return Register();
4324}
4325
4326Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
4327 if ((Subtarget->hasFPARMv8())) {
4328 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
4329 }
4330 return Register();
4331}
4332
4333Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
4334 if ((Subtarget->hasFPRCVT())) {
4335 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
4336 }
4337 if ((Subtarget->hasFPARMv8())) {
4338 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
4339 }
4340 return Register();
4341}
4342
4343Register fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
4344switch (RetVT.SimpleTy) {
4345 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
4346 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
4347 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
4348 default: return Register();
4349}
4350}
4351
4352Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
4353 if ((Subtarget->hasFPRCVT())) {
4354 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
4355 }
4356 if ((Subtarget->hasFullFP16())) {
4357 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
4358 }
4359 return Register();
4360}
4361
4362Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
4363 if ((Subtarget->hasFPRCVT())) {
4364 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
4365 }
4366 if ((Subtarget->hasFPARMv8())) {
4367 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
4368 }
4369 return Register();
4370}
4371
4372Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
4373 if ((Subtarget->hasFPARMv8())) {
4374 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
4375 }
4376 return Register();
4377}
4378
4379Register fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
4380switch (RetVT.SimpleTy) {
4381 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
4382 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
4383 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
4384 default: return Register();
4385}
4386}
4387
4388Register fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
4389 if (RetVT.SimpleTy != MVT::v4f16)
4390 return Register();
4391 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4392 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
4393 }
4394 return Register();
4395}
4396
4397Register fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
4398 if (RetVT.SimpleTy != MVT::v8f16)
4399 return Register();
4400 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4401 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
4402 }
4403 return Register();
4404}
4405
4406Register fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
4407 if (RetVT.SimpleTy != MVT::v2f32)
4408 return Register();
4409 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4410 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
4411 }
4412 return Register();
4413}
4414
4415Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
4416 if (RetVT.SimpleTy != MVT::v4f32)
4417 return Register();
4418 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4419 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
4420 }
4421 return Register();
4422}
4423
4424Register fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
4425 if (RetVT.SimpleTy != MVT::v2f64)
4426 return Register();
4427 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4428 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
4429 }
4430 return Register();
4431}
4432
4433Register fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
4434 switch (VT.SimpleTy) {
4435 case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
4436 case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
4437 case MVT::v4i16: return fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
4438 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
4439 case MVT::v2i32: return fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
4440 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
4441 case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
4442 default: return Register();
4443 }
4444}
4445
4446// FastEmit functions for ISD::SPLAT_VECTOR.
4447
4448Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(Register Op0) {
4449 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4450 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_B, RC: &AArch64::ZPRRegClass, Op0);
4451 }
4452 return Register();
4453}
4454
4455Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(Register Op0) {
4456 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4457 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_H, RC: &AArch64::ZPRRegClass, Op0);
4458 }
4459 return Register();
4460}
4461
4462Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(Register Op0) {
4463 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4464 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_S, RC: &AArch64::ZPRRegClass, Op0);
4465 }
4466 return Register();
4467}
4468
4469Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(MVT RetVT, Register Op0) {
4470switch (RetVT.SimpleTy) {
4471 case MVT::nxv16i8: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(Op0);
4472 case MVT::nxv8i16: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(Op0);
4473 case MVT::nxv4i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(Op0);
4474 default: return Register();
4475}
4476}
4477
4478Register fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(MVT RetVT, Register Op0) {
4479 if (RetVT.SimpleTy != MVT::nxv2i64)
4480 return Register();
4481 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4482 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_D, RC: &AArch64::ZPRRegClass, Op0);
4483 }
4484 return Register();
4485}
4486
4487Register fastEmit_ISD_SPLAT_VECTOR_r(MVT VT, MVT RetVT, Register Op0) {
4488 switch (VT.SimpleTy) {
4489 case MVT::i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(RetVT, Op0);
4490 case MVT::i64: return fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(RetVT, Op0);
4491 default: return Register();
4492 }
4493}
4494
4495// FastEmit functions for ISD::STRICT_FCEIL.
4496
4497Register fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
4498 if (RetVT.SimpleTy != MVT::f16)
4499 return Register();
4500 if ((Subtarget->hasFullFP16())) {
4501 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPHr, RC: &AArch64::FPR16RegClass, Op0);
4502 }
4503 return Register();
4504}
4505
4506Register fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
4507 if (RetVT.SimpleTy != MVT::f32)
4508 return Register();
4509 if ((Subtarget->hasFPARMv8())) {
4510 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPSr, RC: &AArch64::FPR32RegClass, Op0);
4511 }
4512 return Register();
4513}
4514
4515Register fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
4516 if (RetVT.SimpleTy != MVT::f64)
4517 return Register();
4518 if ((Subtarget->hasFPARMv8())) {
4519 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPDr, RC: &AArch64::FPR64RegClass, Op0);
4520 }
4521 return Register();
4522}
4523
4524Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
4525 if (RetVT.SimpleTy != MVT::v4f16)
4526 return Register();
4527 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4528 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f16, RC: &AArch64::FPR64RegClass, Op0);
4529 }
4530 return Register();
4531}
4532
4533Register fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
4534 if (RetVT.SimpleTy != MVT::v8f16)
4535 return Register();
4536 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4537 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv8f16, RC: &AArch64::FPR128RegClass, Op0);
4538 }
4539 return Register();
4540}
4541
4542Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
4543 if (RetVT.SimpleTy != MVT::v2f32)
4544 return Register();
4545 if ((Subtarget->isNeonAvailable())) {
4546 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f32, RC: &AArch64::FPR64RegClass, Op0);
4547 }
4548 return Register();
4549}
4550
4551Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
4552 if (RetVT.SimpleTy != MVT::v4f32)
4553 return Register();
4554 if ((Subtarget->isNeonAvailable())) {
4555 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f32, RC: &AArch64::FPR128RegClass, Op0);
4556 }
4557 return Register();
4558}
4559
4560Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(MVT RetVT, Register Op0) {
4561 if (RetVT.SimpleTy != MVT::v2f64)
4562 return Register();
4563 if ((Subtarget->isNeonAvailable())) {
4564 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f64, RC: &AArch64::FPR128RegClass, Op0);
4565 }
4566 return Register();
4567}
4568
4569Register fastEmit_ISD_STRICT_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
4570 switch (VT.SimpleTy) {
4571 case MVT::f16: return fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(RetVT, Op0);
4572 case MVT::f32: return fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(RetVT, Op0);
4573 case MVT::f64: return fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(RetVT, Op0);
4574 case MVT::v4f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(RetVT, Op0);
4575 case MVT::v8f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(RetVT, Op0);
4576 case MVT::v2f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(RetVT, Op0);
4577 case MVT::v4f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(RetVT, Op0);
4578 case MVT::v2f64: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(RetVT, Op0);
4579 default: return Register();
4580 }
4581}
4582
4583// FastEmit functions for ISD::STRICT_FFLOOR.
4584
4585Register fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
4586 if (RetVT.SimpleTy != MVT::f16)
4587 return Register();
4588 if ((Subtarget->hasFullFP16())) {
4589 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMHr, RC: &AArch64::FPR16RegClass, Op0);
4590 }
4591 return Register();
4592}
4593
4594Register fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
4595 if (RetVT.SimpleTy != MVT::f32)
4596 return Register();
4597 if ((Subtarget->hasFPARMv8())) {
4598 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMSr, RC: &AArch64::FPR32RegClass, Op0);
4599 }
4600 return Register();
4601}
4602
4603Register fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
4604 if (RetVT.SimpleTy != MVT::f64)
4605 return Register();
4606 if ((Subtarget->hasFPARMv8())) {
4607 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMDr, RC: &AArch64::FPR64RegClass, Op0);
4608 }
4609 return Register();
4610}
4611
4612Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
4613 if (RetVT.SimpleTy != MVT::v4f16)
4614 return Register();
4615 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4616 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f16, RC: &AArch64::FPR64RegClass, Op0);
4617 }
4618 return Register();
4619}
4620
4621Register fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
4622 if (RetVT.SimpleTy != MVT::v8f16)
4623 return Register();
4624 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4625 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv8f16, RC: &AArch64::FPR128RegClass, Op0);
4626 }
4627 return Register();
4628}
4629
4630Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
4631 if (RetVT.SimpleTy != MVT::v2f32)
4632 return Register();
4633 if ((Subtarget->isNeonAvailable())) {
4634 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f32, RC: &AArch64::FPR64RegClass, Op0);
4635 }
4636 return Register();
4637}
4638
4639Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
4640 if (RetVT.SimpleTy != MVT::v4f32)
4641 return Register();
4642 if ((Subtarget->isNeonAvailable())) {
4643 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f32, RC: &AArch64::FPR128RegClass, Op0);
4644 }
4645 return Register();
4646}
4647
4648Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(MVT RetVT, Register Op0) {
4649 if (RetVT.SimpleTy != MVT::v2f64)
4650 return Register();
4651 if ((Subtarget->isNeonAvailable())) {
4652 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f64, RC: &AArch64::FPR128RegClass, Op0);
4653 }
4654 return Register();
4655}
4656
4657Register fastEmit_ISD_STRICT_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
4658 switch (VT.SimpleTy) {
4659 case MVT::f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(RetVT, Op0);
4660 case MVT::f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(RetVT, Op0);
4661 case MVT::f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(RetVT, Op0);
4662 case MVT::v4f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(RetVT, Op0);
4663 case MVT::v8f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(RetVT, Op0);
4664 case MVT::v2f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(RetVT, Op0);
4665 case MVT::v4f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(RetVT, Op0);
4666 case MVT::v2f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(RetVT, Op0);
4667 default: return Register();
4668 }
4669}
4670
4671// FastEmit functions for ISD::STRICT_FNEARBYINT.
4672
4673Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
4674 if (RetVT.SimpleTy != MVT::f16)
4675 return Register();
4676 if ((Subtarget->hasFullFP16())) {
4677 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIHr, RC: &AArch64::FPR16RegClass, Op0);
4678 }
4679 return Register();
4680}
4681
4682Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
4683 if (RetVT.SimpleTy != MVT::f32)
4684 return Register();
4685 if ((Subtarget->hasFPARMv8())) {
4686 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTISr, RC: &AArch64::FPR32RegClass, Op0);
4687 }
4688 return Register();
4689}
4690
4691Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
4692 if (RetVT.SimpleTy != MVT::f64)
4693 return Register();
4694 if ((Subtarget->hasFPARMv8())) {
4695 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIDr, RC: &AArch64::FPR64RegClass, Op0);
4696 }
4697 return Register();
4698}
4699
4700Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4701 if (RetVT.SimpleTy != MVT::v4f16)
4702 return Register();
4703 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4704 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f16, RC: &AArch64::FPR64RegClass, Op0);
4705 }
4706 return Register();
4707}
4708
4709Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4710 if (RetVT.SimpleTy != MVT::v8f16)
4711 return Register();
4712 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4713 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv8f16, RC: &AArch64::FPR128RegClass, Op0);
4714 }
4715 return Register();
4716}
4717
4718Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4719 if (RetVT.SimpleTy != MVT::v2f32)
4720 return Register();
4721 if ((Subtarget->isNeonAvailable())) {
4722 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f32, RC: &AArch64::FPR64RegClass, Op0);
4723 }
4724 return Register();
4725}
4726
4727Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4728 if (RetVT.SimpleTy != MVT::v4f32)
4729 return Register();
4730 if ((Subtarget->isNeonAvailable())) {
4731 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f32, RC: &AArch64::FPR128RegClass, Op0);
4732 }
4733 return Register();
4734}
4735
4736Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4737 if (RetVT.SimpleTy != MVT::v2f64)
4738 return Register();
4739 if ((Subtarget->isNeonAvailable())) {
4740 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f64, RC: &AArch64::FPR128RegClass, Op0);
4741 }
4742 return Register();
4743}
4744
4745Register fastEmit_ISD_STRICT_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
4746 switch (VT.SimpleTy) {
4747 case MVT::f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(RetVT, Op0);
4748 case MVT::f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(RetVT, Op0);
4749 case MVT::f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(RetVT, Op0);
4750 case MVT::v4f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(RetVT, Op0);
4751 case MVT::v8f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(RetVT, Op0);
4752 case MVT::v2f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(RetVT, Op0);
4753 case MVT::v4f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(RetVT, Op0);
4754 case MVT::v2f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(RetVT, Op0);
4755 default: return Register();
4756 }
4757}
4758
4759// FastEmit functions for ISD::STRICT_FP_EXTEND.
4760
4761Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(Register Op0) {
4762 if ((Subtarget->hasFPARMv8())) {
4763 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSHr, RC: &AArch64::FPR32RegClass, Op0);
4764 }
4765 return Register();
4766}
4767
4768Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(Register Op0) {
4769 if ((Subtarget->hasFPARMv8())) {
4770 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDHr, RC: &AArch64::FPR64RegClass, Op0);
4771 }
4772 return Register();
4773}
4774
4775Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(MVT RetVT, Register Op0) {
4776switch (RetVT.SimpleTy) {
4777 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(Op0);
4778 case MVT::f64: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(Op0);
4779 default: return Register();
4780}
4781}
4782
4783Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
4784 if (RetVT.SimpleTy != MVT::f64)
4785 return Register();
4786 if ((Subtarget->hasFPARMv8())) {
4787 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDSr, RC: &AArch64::FPR64RegClass, Op0);
4788 }
4789 return Register();
4790}
4791
4792Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
4793 if (RetVT.SimpleTy != MVT::v4f32)
4794 return Register();
4795 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv4i16, RC: &AArch64::FPR128RegClass, Op0);
4796}
4797
4798Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4bf16_r(MVT RetVT, Register Op0) {
4799 if (RetVT.SimpleTy != MVT::v4f32)
4800 return Register();
4801 if ((Subtarget->isNeonAvailable())) {
4802 return fastEmitInst_r(MachineInstOpcode: AArch64::SHLLv4i16, RC: &AArch64::FPR128RegClass, Op0);
4803 }
4804 return Register();
4805}
4806
4807Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(MVT RetVT, Register Op0) {
4808 if (RetVT.SimpleTy != MVT::v2f64)
4809 return Register();
4810 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv2i32, RC: &AArch64::FPR128RegClass, Op0);
4811}
4812
4813Register fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
4814 switch (VT.SimpleTy) {
4815 case MVT::f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(RetVT, Op0);
4816 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0);
4817 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
4818 case MVT::v4bf16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4bf16_r(RetVT, Op0);
4819 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(RetVT, Op0);
4820 default: return Register();
4821 }
4822}
4823
4824// FastEmit functions for ISD::STRICT_FP_ROUND.
4825
4826Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_bf16_r(Register Op0) {
4827 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
4828 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVT, RC: &AArch64::FPR16RegClass, Op0);
4829 }
4830 return Register();
4831}
4832
4833Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_f16_r(Register Op0) {
4834 if ((Subtarget->hasFPARMv8())) {
4835 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHSr, RC: &AArch64::FPR16RegClass, Op0);
4836 }
4837 return Register();
4838}
4839
4840Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4841switch (RetVT.SimpleTy) {
4842 case MVT::bf16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_bf16_r(Op0);
4843 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_f16_r(Op0);
4844 default: return Register();
4845}
4846}
4847
4848Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(Register Op0) {
4849 if ((Subtarget->hasFPARMv8())) {
4850 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHDr, RC: &AArch64::FPR16RegClass, Op0);
4851 }
4852 return Register();
4853}
4854
4855Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(Register Op0) {
4856 if ((Subtarget->hasFPARMv8())) {
4857 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSDr, RC: &AArch64::FPR32RegClass, Op0);
4858 }
4859 return Register();
4860}
4861
4862Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4863switch (RetVT.SimpleTy) {
4864 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(Op0);
4865 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(Op0);
4866 default: return Register();
4867}
4868}
4869
4870Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Register Op0) {
4871 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
4872}
4873
4874Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Register Op0) {
4875 if ((Subtarget->hasBF16()) && (Subtarget->isNeonAvailable())) {
4876 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVTN, RC: &AArch64::FPR64RegClass, Op0);
4877 }
4878 return Register();
4879}
4880
4881Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
4882switch (RetVT.SimpleTy) {
4883 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Op0);
4884 case MVT::v4bf16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Op0);
4885 default: return Register();
4886}
4887}
4888
4889Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
4890 if (RetVT.SimpleTy != MVT::v2f32)
4891 return Register();
4892 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
4893}
4894
4895Register fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
4896 switch (VT.SimpleTy) {
4897 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(RetVT, Op0);
4898 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0);
4899 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
4900 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(RetVT, Op0);
4901 default: return Register();
4902 }
4903}
4904
4905// FastEmit functions for ISD::STRICT_FP_TO_SINT.
4906
4907Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
4908 if ((Subtarget->hasFPRCVT())) {
4909 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSHr, RC: &AArch64::FPR32RegClass, Op0);
4910 }
4911 if ((Subtarget->hasFullFP16())) {
4912 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
4913 }
4914 return Register();
4915}
4916
4917Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
4918 if ((Subtarget->hasFPRCVT())) {
4919 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDHr, RC: &AArch64::FPR64RegClass, Op0);
4920 }
4921 if ((Subtarget->hasFullFP16())) {
4922 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
4923 }
4924 return Register();
4925}
4926
4927Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
4928switch (RetVT.SimpleTy) {
4929 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
4930 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
4931 default: return Register();
4932}
4933}
4934
4935Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
4936 if ((Subtarget->hasFPARMv8())) {
4937 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
4938 }
4939 return Register();
4940}
4941
4942Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
4943 if ((Subtarget->hasFPRCVT())) {
4944 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDSr, RC: &AArch64::FPR64RegClass, Op0);
4945 }
4946 if ((Subtarget->hasFPARMv8())) {
4947 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
4948 }
4949 return Register();
4950}
4951
4952Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
4953switch (RetVT.SimpleTy) {
4954 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
4955 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
4956 default: return Register();
4957}
4958}
4959
4960Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
4961 if ((Subtarget->hasFPRCVT())) {
4962 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSDr, RC: &AArch64::FPR32RegClass, Op0);
4963 }
4964 if ((Subtarget->hasFPARMv8())) {
4965 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
4966 }
4967 return Register();
4968}
4969
4970Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
4971 if ((Subtarget->hasFPARMv8())) {
4972 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
4973 }
4974 return Register();
4975}
4976
4977Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
4978switch (RetVT.SimpleTy) {
4979 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
4980 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
4981 default: return Register();
4982}
4983}
4984
4985Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4986 if (RetVT.SimpleTy != MVT::v4i16)
4987 return Register();
4988 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4989 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
4990 }
4991 return Register();
4992}
4993
4994Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4995 if (RetVT.SimpleTy != MVT::v8i16)
4996 return Register();
4997 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4998 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
4999 }
5000 return Register();
5001}
5002
5003Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
5004 if (RetVT.SimpleTy != MVT::v2i32)
5005 return Register();
5006 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5007 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
5008 }
5009 return Register();
5010}
5011
5012Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
5013 if (RetVT.SimpleTy != MVT::v4i32)
5014 return Register();
5015 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5016 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
5017 }
5018 return Register();
5019}
5020
5021Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
5022 if (RetVT.SimpleTy != MVT::v2i64)
5023 return Register();
5024 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5025 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
5026 }
5027 return Register();
5028}
5029
5030Register fastEmit_ISD_STRICT_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
5031 switch (VT.SimpleTy) {
5032 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
5033 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
5034 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
5035 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
5036 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
5037 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
5038 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
5039 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0);
5040 default: return Register();
5041 }
5042}
5043
5044// FastEmit functions for ISD::STRICT_FP_TO_UINT.
5045
5046Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
5047 if ((Subtarget->hasFPRCVT())) {
5048 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSHr, RC: &AArch64::FPR32RegClass, Op0);
5049 }
5050 if ((Subtarget->hasFullFP16())) {
5051 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
5052 }
5053 return Register();
5054}
5055
5056Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
5057 if ((Subtarget->hasFPRCVT())) {
5058 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDHr, RC: &AArch64::FPR64RegClass, Op0);
5059 }
5060 if ((Subtarget->hasFullFP16())) {
5061 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
5062 }
5063 return Register();
5064}
5065
5066Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
5067switch (RetVT.SimpleTy) {
5068 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
5069 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
5070 default: return Register();
5071}
5072}
5073
5074Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
5075 if ((Subtarget->hasFPARMv8())) {
5076 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
5077 }
5078 return Register();
5079}
5080
5081Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
5082 if ((Subtarget->hasFPRCVT())) {
5083 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDSr, RC: &AArch64::FPR64RegClass, Op0);
5084 }
5085 if ((Subtarget->hasFPARMv8())) {
5086 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
5087 }
5088 return Register();
5089}
5090
5091Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
5092switch (RetVT.SimpleTy) {
5093 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
5094 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
5095 default: return Register();
5096}
5097}
5098
5099Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
5100 if ((Subtarget->hasFPRCVT())) {
5101 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSDr, RC: &AArch64::FPR32RegClass, Op0);
5102 }
5103 if ((Subtarget->hasFPARMv8())) {
5104 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
5105 }
5106 return Register();
5107}
5108
5109Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
5110 if ((Subtarget->hasFPARMv8())) {
5111 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
5112 }
5113 return Register();
5114}
5115
5116Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
5117switch (RetVT.SimpleTy) {
5118 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
5119 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
5120 default: return Register();
5121}
5122}
5123
5124Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
5125 if (RetVT.SimpleTy != MVT::v4i16)
5126 return Register();
5127 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5128 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
5129 }
5130 return Register();
5131}
5132
5133Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
5134 if (RetVT.SimpleTy != MVT::v8i16)
5135 return Register();
5136 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5137 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
5138 }
5139 return Register();
5140}
5141
5142Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
5143 if (RetVT.SimpleTy != MVT::v2i32)
5144 return Register();
5145 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5146 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
5147 }
5148 return Register();
5149}
5150
5151Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
5152 if (RetVT.SimpleTy != MVT::v4i32)
5153 return Register();
5154 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5155 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
5156 }
5157 return Register();
5158}
5159
5160Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
5161 if (RetVT.SimpleTy != MVT::v2i64)
5162 return Register();
5163 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5164 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
5165 }
5166 return Register();
5167}
5168
5169Register fastEmit_ISD_STRICT_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
5170 switch (VT.SimpleTy) {
5171 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
5172 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
5173 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
5174 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
5175 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
5176 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
5177 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
5178 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0);
5179 default: return Register();
5180 }
5181}
5182
5183// FastEmit functions for ISD::STRICT_FRINT.
5184
5185Register fastEmit_ISD_STRICT_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
5186 if (RetVT.SimpleTy != MVT::f16)
5187 return Register();
5188 if ((Subtarget->hasFullFP16())) {
5189 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXHr, RC: &AArch64::FPR16RegClass, Op0);
5190 }
5191 return Register();
5192}
5193
5194Register fastEmit_ISD_STRICT_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
5195 if (RetVT.SimpleTy != MVT::f32)
5196 return Register();
5197 if ((Subtarget->hasFPARMv8())) {
5198 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXSr, RC: &AArch64::FPR32RegClass, Op0);
5199 }
5200 return Register();
5201}
5202
5203Register fastEmit_ISD_STRICT_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
5204 if (RetVT.SimpleTy != MVT::f64)
5205 return Register();
5206 if ((Subtarget->hasFPARMv8())) {
5207 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXDr, RC: &AArch64::FPR64RegClass, Op0);
5208 }
5209 return Register();
5210}
5211
5212Register fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
5213 if (RetVT.SimpleTy != MVT::v4f16)
5214 return Register();
5215 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5216 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f16, RC: &AArch64::FPR64RegClass, Op0);
5217 }
5218 return Register();
5219}
5220
5221Register fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
5222 if (RetVT.SimpleTy != MVT::v8f16)
5223 return Register();
5224 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5225 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv8f16, RC: &AArch64::FPR128RegClass, Op0);
5226 }
5227 return Register();
5228}
5229
5230Register fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
5231 if (RetVT.SimpleTy != MVT::v2f32)
5232 return Register();
5233 if ((Subtarget->isNeonAvailable())) {
5234 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f32, RC: &AArch64::FPR64RegClass, Op0);
5235 }
5236 return Register();
5237}
5238
5239Register fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
5240 if (RetVT.SimpleTy != MVT::v4f32)
5241 return Register();
5242 if ((Subtarget->isNeonAvailable())) {
5243 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f32, RC: &AArch64::FPR128RegClass, Op0);
5244 }
5245 return Register();
5246}
5247
5248Register fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
5249 if (RetVT.SimpleTy != MVT::v2f64)
5250 return Register();
5251 if ((Subtarget->isNeonAvailable())) {
5252 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f64, RC: &AArch64::FPR128RegClass, Op0);
5253 }
5254 return Register();
5255}
5256
5257Register fastEmit_ISD_STRICT_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
5258 switch (VT.SimpleTy) {
5259 case MVT::f16: return fastEmit_ISD_STRICT_FRINT_MVT_f16_r(RetVT, Op0);
5260 case MVT::f32: return fastEmit_ISD_STRICT_FRINT_MVT_f32_r(RetVT, Op0);
5261 case MVT::f64: return fastEmit_ISD_STRICT_FRINT_MVT_f64_r(RetVT, Op0);
5262 case MVT::v4f16: return fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(RetVT, Op0);
5263 case MVT::v8f16: return fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(RetVT, Op0);
5264 case MVT::v2f32: return fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(RetVT, Op0);
5265 case MVT::v4f32: return fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(RetVT, Op0);
5266 case MVT::v2f64: return fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(RetVT, Op0);
5267 default: return Register();
5268 }
5269}
5270
5271// FastEmit functions for ISD::STRICT_FROUND.
5272
5273Register fastEmit_ISD_STRICT_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5274 if (RetVT.SimpleTy != MVT::f16)
5275 return Register();
5276 if ((Subtarget->hasFullFP16())) {
5277 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAHr, RC: &AArch64::FPR16RegClass, Op0);
5278 }
5279 return Register();
5280}
5281
5282Register fastEmit_ISD_STRICT_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5283 if (RetVT.SimpleTy != MVT::f32)
5284 return Register();
5285 if ((Subtarget->hasFPARMv8())) {
5286 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTASr, RC: &AArch64::FPR32RegClass, Op0);
5287 }
5288 return Register();
5289}
5290
5291Register fastEmit_ISD_STRICT_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5292 if (RetVT.SimpleTy != MVT::f64)
5293 return Register();
5294 if ((Subtarget->hasFPARMv8())) {
5295 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTADr, RC: &AArch64::FPR64RegClass, Op0);
5296 }
5297 return Register();
5298}
5299
5300Register fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
5301 if (RetVT.SimpleTy != MVT::v4f16)
5302 return Register();
5303 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5304 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f16, RC: &AArch64::FPR64RegClass, Op0);
5305 }
5306 return Register();
5307}
5308
5309Register fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
5310 if (RetVT.SimpleTy != MVT::v8f16)
5311 return Register();
5312 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5313 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv8f16, RC: &AArch64::FPR128RegClass, Op0);
5314 }
5315 return Register();
5316}
5317
5318Register fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
5319 if (RetVT.SimpleTy != MVT::v2f32)
5320 return Register();
5321 if ((Subtarget->isNeonAvailable())) {
5322 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f32, RC: &AArch64::FPR64RegClass, Op0);
5323 }
5324 return Register();
5325}
5326
5327Register fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
5328 if (RetVT.SimpleTy != MVT::v4f32)
5329 return Register();
5330 if ((Subtarget->isNeonAvailable())) {
5331 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f32, RC: &AArch64::FPR128RegClass, Op0);
5332 }
5333 return Register();
5334}
5335
5336Register fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
5337 if (RetVT.SimpleTy != MVT::v2f64)
5338 return Register();
5339 if ((Subtarget->isNeonAvailable())) {
5340 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f64, RC: &AArch64::FPR128RegClass, Op0);
5341 }
5342 return Register();
5343}
5344
5345Register fastEmit_ISD_STRICT_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
5346 switch (VT.SimpleTy) {
5347 case MVT::f16: return fastEmit_ISD_STRICT_FROUND_MVT_f16_r(RetVT, Op0);
5348 case MVT::f32: return fastEmit_ISD_STRICT_FROUND_MVT_f32_r(RetVT, Op0);
5349 case MVT::f64: return fastEmit_ISD_STRICT_FROUND_MVT_f64_r(RetVT, Op0);
5350 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(RetVT, Op0);
5351 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(RetVT, Op0);
5352 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(RetVT, Op0);
5353 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(RetVT, Op0);
5354 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(RetVT, Op0);
5355 default: return Register();
5356 }
5357}
5358
5359// FastEmit functions for ISD::STRICT_FROUNDEVEN.
5360
5361Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
5362 if (RetVT.SimpleTy != MVT::f16)
5363 return Register();
5364 if ((Subtarget->hasFullFP16())) {
5365 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNHr, RC: &AArch64::FPR16RegClass, Op0);
5366 }
5367 return Register();
5368}
5369
5370Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
5371 if (RetVT.SimpleTy != MVT::f32)
5372 return Register();
5373 if ((Subtarget->hasFPARMv8())) {
5374 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNSr, RC: &AArch64::FPR32RegClass, Op0);
5375 }
5376 return Register();
5377}
5378
5379Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
5380 if (RetVT.SimpleTy != MVT::f64)
5381 return Register();
5382 if ((Subtarget->hasFPARMv8())) {
5383 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNDr, RC: &AArch64::FPR64RegClass, Op0);
5384 }
5385 return Register();
5386}
5387
5388Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
5389 if (RetVT.SimpleTy != MVT::v4f16)
5390 return Register();
5391 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5392 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f16, RC: &AArch64::FPR64RegClass, Op0);
5393 }
5394 return Register();
5395}
5396
5397Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
5398 if (RetVT.SimpleTy != MVT::v8f16)
5399 return Register();
5400 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5401 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv8f16, RC: &AArch64::FPR128RegClass, Op0);
5402 }
5403 return Register();
5404}
5405
5406Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
5407 if (RetVT.SimpleTy != MVT::v2f32)
5408 return Register();
5409 if ((Subtarget->isNeonAvailable())) {
5410 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f32, RC: &AArch64::FPR64RegClass, Op0);
5411 }
5412 return Register();
5413}
5414
5415Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
5416 if (RetVT.SimpleTy != MVT::v4f32)
5417 return Register();
5418 if ((Subtarget->isNeonAvailable())) {
5419 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f32, RC: &AArch64::FPR128RegClass, Op0);
5420 }
5421 return Register();
5422}
5423
5424Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, Register Op0) {
5425 if (RetVT.SimpleTy != MVT::v2f64)
5426 return Register();
5427 if ((Subtarget->isNeonAvailable())) {
5428 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f64, RC: &AArch64::FPR128RegClass, Op0);
5429 }
5430 return Register();
5431}
5432
5433Register fastEmit_ISD_STRICT_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
5434 switch (VT.SimpleTy) {
5435 case MVT::f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
5436 case MVT::f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
5437 case MVT::f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
5438 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
5439 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
5440 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
5441 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
5442 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0);
5443 default: return Register();
5444 }
5445}
5446
5447// FastEmit functions for ISD::STRICT_FSQRT.
5448
5449Register fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
5450 if (RetVT.SimpleTy != MVT::f16)
5451 return Register();
5452 if ((Subtarget->hasFullFP16())) {
5453 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTHr, RC: &AArch64::FPR16RegClass, Op0);
5454 }
5455 return Register();
5456}
5457
5458Register fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
5459 if (RetVT.SimpleTy != MVT::f32)
5460 return Register();
5461 if ((Subtarget->hasFPARMv8())) {
5462 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTSr, RC: &AArch64::FPR32RegClass, Op0);
5463 }
5464 return Register();
5465}
5466
5467Register fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
5468 if (RetVT.SimpleTy != MVT::f64)
5469 return Register();
5470 if ((Subtarget->hasFPARMv8())) {
5471 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTDr, RC: &AArch64::FPR64RegClass, Op0);
5472 }
5473 return Register();
5474}
5475
5476Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(MVT RetVT, Register Op0) {
5477 if (RetVT.SimpleTy != MVT::v4f16)
5478 return Register();
5479 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5480 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f16, RC: &AArch64::FPR64RegClass, Op0);
5481 }
5482 return Register();
5483}
5484
5485Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
5486 if (RetVT.SimpleTy != MVT::v8f16)
5487 return Register();
5488 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5489 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv8f16, RC: &AArch64::FPR128RegClass, Op0);
5490 }
5491 return Register();
5492}
5493
5494Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(MVT RetVT, Register Op0) {
5495 if (RetVT.SimpleTy != MVT::v2f32)
5496 return Register();
5497 if ((Subtarget->isNeonAvailable())) {
5498 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f32, RC: &AArch64::FPR64RegClass, Op0);
5499 }
5500 return Register();
5501}
5502
5503Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
5504 if (RetVT.SimpleTy != MVT::v4f32)
5505 return Register();
5506 if ((Subtarget->isNeonAvailable())) {
5507 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f32, RC: &AArch64::FPR128RegClass, Op0);
5508 }
5509 return Register();
5510}
5511
5512Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
5513 if (RetVT.SimpleTy != MVT::v2f64)
5514 return Register();
5515 if ((Subtarget->isNeonAvailable())) {
5516 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f64, RC: &AArch64::FPR128RegClass, Op0);
5517 }
5518 return Register();
5519}
5520
5521Register fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
5522 switch (VT.SimpleTy) {
5523 case MVT::f16: return fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(RetVT, Op0);
5524 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0);
5525 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0);
5526 case MVT::v4f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(RetVT, Op0);
5527 case MVT::v8f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(RetVT, Op0);
5528 case MVT::v2f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(RetVT, Op0);
5529 case MVT::v4f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(RetVT, Op0);
5530 case MVT::v2f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(RetVT, Op0);
5531 default: return Register();
5532 }
5533}
5534
5535// FastEmit functions for ISD::STRICT_FTRUNC.
5536
5537Register fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
5538 if (RetVT.SimpleTy != MVT::f16)
5539 return Register();
5540 if ((Subtarget->hasFullFP16())) {
5541 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZHr, RC: &AArch64::FPR16RegClass, Op0);
5542 }
5543 return Register();
5544}
5545
5546Register fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
5547 if (RetVT.SimpleTy != MVT::f32)
5548 return Register();
5549 if ((Subtarget->hasFPARMv8())) {
5550 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZSr, RC: &AArch64::FPR32RegClass, Op0);
5551 }
5552 return Register();
5553}
5554
5555Register fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
5556 if (RetVT.SimpleTy != MVT::f64)
5557 return Register();
5558 if ((Subtarget->hasFPARMv8())) {
5559 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZDr, RC: &AArch64::FPR64RegClass, Op0);
5560 }
5561 return Register();
5562}
5563
5564Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
5565 if (RetVT.SimpleTy != MVT::v4f16)
5566 return Register();
5567 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5568 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f16, RC: &AArch64::FPR64RegClass, Op0);
5569 }
5570 return Register();
5571}
5572
5573Register fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
5574 if (RetVT.SimpleTy != MVT::v8f16)
5575 return Register();
5576 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5577 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv8f16, RC: &AArch64::FPR128RegClass, Op0);
5578 }
5579 return Register();
5580}
5581
5582Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
5583 if (RetVT.SimpleTy != MVT::v2f32)
5584 return Register();
5585 if ((Subtarget->isNeonAvailable())) {
5586 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f32, RC: &AArch64::FPR64RegClass, Op0);
5587 }
5588 return Register();
5589}
5590
5591Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
5592 if (RetVT.SimpleTy != MVT::v4f32)
5593 return Register();
5594 if ((Subtarget->isNeonAvailable())) {
5595 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f32, RC: &AArch64::FPR128RegClass, Op0);
5596 }
5597 return Register();
5598}
5599
5600Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(MVT RetVT, Register Op0) {
5601 if (RetVT.SimpleTy != MVT::v2f64)
5602 return Register();
5603 if ((Subtarget->isNeonAvailable())) {
5604 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f64, RC: &AArch64::FPR128RegClass, Op0);
5605 }
5606 return Register();
5607}
5608
5609Register fastEmit_ISD_STRICT_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
5610 switch (VT.SimpleTy) {
5611 case MVT::f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(RetVT, Op0);
5612 case MVT::f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(RetVT, Op0);
5613 case MVT::f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(RetVT, Op0);
5614 case MVT::v4f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(RetVT, Op0);
5615 case MVT::v8f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(RetVT, Op0);
5616 case MVT::v2f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(RetVT, Op0);
5617 case MVT::v4f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(RetVT, Op0);
5618 case MVT::v2f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(RetVT, Op0);
5619 default: return Register();
5620 }
5621}
5622
5623// FastEmit functions for ISD::STRICT_LLROUND.
5624
5625Register fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5626 if (RetVT.SimpleTy != MVT::i64)
5627 return Register();
5628 if ((Subtarget->hasFullFP16())) {
5629 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
5630 }
5631 return Register();
5632}
5633
5634Register fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5635 if (RetVT.SimpleTy != MVT::i64)
5636 return Register();
5637 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
5638}
5639
5640Register fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5641 if (RetVT.SimpleTy != MVT::i64)
5642 return Register();
5643 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
5644}
5645
5646Register fastEmit_ISD_STRICT_LLROUND_r(MVT VT, MVT RetVT, Register Op0) {
5647 switch (VT.SimpleTy) {
5648 case MVT::f16: return fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(RetVT, Op0);
5649 case MVT::f32: return fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(RetVT, Op0);
5650 case MVT::f64: return fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(RetVT, Op0);
5651 default: return Register();
5652 }
5653}
5654
5655// FastEmit functions for ISD::STRICT_LROUND.
5656
5657Register fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(Register Op0) {
5658 if ((Subtarget->hasFullFP16())) {
5659 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWHr, RC: &AArch64::GPR32RegClass, Op0);
5660 }
5661 return Register();
5662}
5663
5664Register fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(Register Op0) {
5665 if ((Subtarget->hasFullFP16())) {
5666 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
5667 }
5668 return Register();
5669}
5670
5671Register fastEmit_ISD_STRICT_LROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5672switch (RetVT.SimpleTy) {
5673 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(Op0);
5674 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(Op0);
5675 default: return Register();
5676}
5677}
5678
5679Register fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(Register Op0) {
5680 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWSr, RC: &AArch64::GPR32RegClass, Op0);
5681}
5682
5683Register fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(Register Op0) {
5684 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
5685}
5686
5687Register fastEmit_ISD_STRICT_LROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5688switch (RetVT.SimpleTy) {
5689 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(Op0);
5690 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(Op0);
5691 default: return Register();
5692}
5693}
5694
5695Register fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(Register Op0) {
5696 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWDr, RC: &AArch64::GPR32RegClass, Op0);
5697}
5698
5699Register fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(Register Op0) {
5700 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
5701}
5702
5703Register fastEmit_ISD_STRICT_LROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5704switch (RetVT.SimpleTy) {
5705 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(Op0);
5706 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(Op0);
5707 default: return Register();
5708}
5709}
5710
5711Register fastEmit_ISD_STRICT_LROUND_r(MVT VT, MVT RetVT, Register Op0) {
5712 switch (VT.SimpleTy) {
5713 case MVT::f16: return fastEmit_ISD_STRICT_LROUND_MVT_f16_r(RetVT, Op0);
5714 case MVT::f32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_r(RetVT, Op0);
5715 case MVT::f64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_r(RetVT, Op0);
5716 default: return Register();
5717 }
5718}
5719
5720// FastEmit functions for ISD::STRICT_SINT_TO_FP.
5721
5722Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5723 if ((Subtarget->hasFPRCVT())) {
5724 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5725 }
5726 if ((Subtarget->hasFullFP16())) {
5727 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5728 }
5729 return Register();
5730}
5731
5732Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5733 if ((Subtarget->hasFPARMv8())) {
5734 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5735 }
5736 return Register();
5737}
5738
5739Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5740 if ((Subtarget->hasFPRCVT())) {
5741 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5742 }
5743 if ((Subtarget->hasFPARMv8())) {
5744 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5745 }
5746 return Register();
5747}
5748
5749Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5750switch (RetVT.SimpleTy) {
5751 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5752 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5753 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5754 default: return Register();
5755}
5756}
5757
5758Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5759 if ((Subtarget->hasFPRCVT())) {
5760 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5761 }
5762 if ((Subtarget->hasFullFP16())) {
5763 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5764 }
5765 return Register();
5766}
5767
5768Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5769 if ((Subtarget->hasFPRCVT())) {
5770 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5771 }
5772 if ((Subtarget->hasFPARMv8())) {
5773 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5774 }
5775 return Register();
5776}
5777
5778Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5779 if ((Subtarget->hasFPARMv8())) {
5780 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5781 }
5782 return Register();
5783}
5784
5785Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5786switch (RetVT.SimpleTy) {
5787 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5788 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5789 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5790 default: return Register();
5791}
5792}
5793
5794Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5795 if (RetVT.SimpleTy != MVT::v4f16)
5796 return Register();
5797 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5798 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5799 }
5800 return Register();
5801}
5802
5803Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5804 if (RetVT.SimpleTy != MVT::v8f16)
5805 return Register();
5806 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5807 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5808 }
5809 return Register();
5810}
5811
5812Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5813 if (RetVT.SimpleTy != MVT::v2f32)
5814 return Register();
5815 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5816 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5817 }
5818 return Register();
5819}
5820
5821Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5822 if (RetVT.SimpleTy != MVT::v4f32)
5823 return Register();
5824 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5825 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5826 }
5827 return Register();
5828}
5829
5830Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5831 if (RetVT.SimpleTy != MVT::v2f64)
5832 return Register();
5833 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5834 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5835 }
5836 return Register();
5837}
5838
5839Register fastEmit_ISD_STRICT_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
5840 switch (VT.SimpleTy) {
5841 case MVT::i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
5842 case MVT::i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
5843 case MVT::v4i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
5844 case MVT::v8i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
5845 case MVT::v2i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
5846 case MVT::v4i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
5847 case MVT::v2i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
5848 default: return Register();
5849 }
5850}
5851
5852// FastEmit functions for ISD::STRICT_UINT_TO_FP.
5853
5854Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5855 if ((Subtarget->hasFPRCVT())) {
5856 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5857 }
5858 if ((Subtarget->hasFullFP16())) {
5859 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5860 }
5861 return Register();
5862}
5863
5864Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5865 if ((Subtarget->hasFPARMv8())) {
5866 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5867 }
5868 return Register();
5869}
5870
5871Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5872 if ((Subtarget->hasFPRCVT())) {
5873 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5874 }
5875 if ((Subtarget->hasFPARMv8())) {
5876 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5877 }
5878 return Register();
5879}
5880
5881Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5882switch (RetVT.SimpleTy) {
5883 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5884 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5885 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5886 default: return Register();
5887}
5888}
5889
5890Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5891 if ((Subtarget->hasFPRCVT())) {
5892 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5893 }
5894 if ((Subtarget->hasFullFP16())) {
5895 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5896 }
5897 return Register();
5898}
5899
5900Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5901 if ((Subtarget->hasFPRCVT())) {
5902 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5903 }
5904 if ((Subtarget->hasFPARMv8())) {
5905 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5906 }
5907 return Register();
5908}
5909
5910Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5911 if ((Subtarget->hasFPARMv8())) {
5912 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5913 }
5914 return Register();
5915}
5916
5917Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5918switch (RetVT.SimpleTy) {
5919 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5920 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5921 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5922 default: return Register();
5923}
5924}
5925
5926Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5927 if (RetVT.SimpleTy != MVT::v4f16)
5928 return Register();
5929 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5930 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5931 }
5932 return Register();
5933}
5934
5935Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5936 if (RetVT.SimpleTy != MVT::v8f16)
5937 return Register();
5938 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5939 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5940 }
5941 return Register();
5942}
5943
5944Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5945 if (RetVT.SimpleTy != MVT::v2f32)
5946 return Register();
5947 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5948 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5949 }
5950 return Register();
5951}
5952
5953Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5954 if (RetVT.SimpleTy != MVT::v4f32)
5955 return Register();
5956 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5957 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5958 }
5959 return Register();
5960}
5961
5962Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5963 if (RetVT.SimpleTy != MVT::v2f64)
5964 return Register();
5965 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5966 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5967 }
5968 return Register();
5969}
5970
5971Register fastEmit_ISD_STRICT_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
5972 switch (VT.SimpleTy) {
5973 case MVT::i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(RetVT, Op0);
5974 case MVT::i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(RetVT, Op0);
5975 case MVT::v4i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
5976 case MVT::v8i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
5977 case MVT::v2i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
5978 case MVT::v4i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
5979 case MVT::v2i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
5980 default: return Register();
5981 }
5982}
5983
5984// FastEmit functions for ISD::TRUNCATE.
5985
5986Register fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, Register Op0) {
5987 if (RetVT.SimpleTy != MVT::i32)
5988 return Register();
5989 return fastEmitInst_extractsubreg(RetVT, Op0, Idx: AArch64::sub_32);
5990}
5991
5992Register fastEmit_ISD_TRUNCATE_MVT_v8i16_r(MVT RetVT, Register Op0) {
5993 if (RetVT.SimpleTy != MVT::v8i8)
5994 return Register();
5995 if ((Subtarget->isNeonAvailable())) {
5996 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5997 }
5998 return Register();
5999}
6000
6001Register fastEmit_ISD_TRUNCATE_MVT_v4i32_r(MVT RetVT, Register Op0) {
6002 if (RetVT.SimpleTy != MVT::v4i16)
6003 return Register();
6004 if ((Subtarget->isNeonAvailable())) {
6005 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
6006 }
6007 return Register();
6008}
6009
6010Register fastEmit_ISD_TRUNCATE_MVT_v2i64_r(MVT RetVT, Register Op0) {
6011 if (RetVT.SimpleTy != MVT::v2i32)
6012 return Register();
6013 if ((Subtarget->isNeonAvailable())) {
6014 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
6015 }
6016 return Register();
6017}
6018
6019Register fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, Register Op0) {
6020 switch (VT.SimpleTy) {
6021 case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0);
6022 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i16_r(RetVT, Op0);
6023 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_MVT_v4i32_r(RetVT, Op0);
6024 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_MVT_v2i64_r(RetVT, Op0);
6025 default: return Register();
6026 }
6027}
6028
6029// FastEmit functions for ISD::TRUNCATE_SSAT_S.
6030
6031Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v8i16_r(MVT RetVT, Register Op0) {
6032 if (RetVT.SimpleTy != MVT::v8i8)
6033 return Register();
6034 if ((Subtarget->isNeonAvailable())) {
6035 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
6036 }
6037 return Register();
6038}
6039
6040Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v4i32_r(MVT RetVT, Register Op0) {
6041 if (RetVT.SimpleTy != MVT::v4i16)
6042 return Register();
6043 if ((Subtarget->isNeonAvailable())) {
6044 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
6045 }
6046 return Register();
6047}
6048
6049Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v2i64_r(MVT RetVT, Register Op0) {
6050 if (RetVT.SimpleTy != MVT::v2i32)
6051 return Register();
6052 if ((Subtarget->isNeonAvailable())) {
6053 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
6054 }
6055 return Register();
6056}
6057
6058Register fastEmit_ISD_TRUNCATE_SSAT_S_r(MVT VT, MVT RetVT, Register Op0) {
6059 switch (VT.SimpleTy) {
6060 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v8i16_r(RetVT, Op0);
6061 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v4i32_r(RetVT, Op0);
6062 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v2i64_r(RetVT, Op0);
6063 default: return Register();
6064 }
6065}
6066
6067// FastEmit functions for ISD::TRUNCATE_SSAT_U.
6068
6069Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v8i16_r(MVT RetVT, Register Op0) {
6070 if (RetVT.SimpleTy != MVT::v8i8)
6071 return Register();
6072 if ((Subtarget->isNeonAvailable())) {
6073 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv8i8, RC: &AArch64::FPR64RegClass, Op0);
6074 }
6075 return Register();
6076}
6077
6078Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v4i32_r(MVT RetVT, Register Op0) {
6079 if (RetVT.SimpleTy != MVT::v4i16)
6080 return Register();
6081 if ((Subtarget->isNeonAvailable())) {
6082 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv4i16, RC: &AArch64::FPR64RegClass, Op0);
6083 }
6084 return Register();
6085}
6086
6087Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v2i64_r(MVT RetVT, Register Op0) {
6088 if (RetVT.SimpleTy != MVT::v2i32)
6089 return Register();
6090 if ((Subtarget->isNeonAvailable())) {
6091 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv2i32, RC: &AArch64::FPR64RegClass, Op0);
6092 }
6093 return Register();
6094}
6095
6096Register fastEmit_ISD_TRUNCATE_SSAT_U_r(MVT VT, MVT RetVT, Register Op0) {
6097 switch (VT.SimpleTy) {
6098 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v8i16_r(RetVT, Op0);
6099 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v4i32_r(RetVT, Op0);
6100 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v2i64_r(RetVT, Op0);
6101 default: return Register();
6102 }
6103}
6104
6105// FastEmit functions for ISD::TRUNCATE_USAT_U.
6106
6107Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v8i16_r(MVT RetVT, Register Op0) {
6108 if (RetVT.SimpleTy != MVT::v8i8)
6109 return Register();
6110 if ((Subtarget->isNeonAvailable())) {
6111 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
6112 }
6113 return Register();
6114}
6115
6116Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v4i32_r(MVT RetVT, Register Op0) {
6117 if (RetVT.SimpleTy != MVT::v4i16)
6118 return Register();
6119 if ((Subtarget->isNeonAvailable())) {
6120 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
6121 }
6122 return Register();
6123}
6124
6125Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v2i64_r(MVT RetVT, Register Op0) {
6126 if (RetVT.SimpleTy != MVT::v2i32)
6127 return Register();
6128 if ((Subtarget->isNeonAvailable())) {
6129 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
6130 }
6131 return Register();
6132}
6133
6134Register fastEmit_ISD_TRUNCATE_USAT_U_r(MVT VT, MVT RetVT, Register Op0) {
6135 switch (VT.SimpleTy) {
6136 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v8i16_r(RetVT, Op0);
6137 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v4i32_r(RetVT, Op0);
6138 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v2i64_r(RetVT, Op0);
6139 default: return Register();
6140 }
6141}
6142
6143// FastEmit functions for ISD::UINT_TO_FP.
6144
6145Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
6146 if ((Subtarget->hasFPRCVT())) {
6147 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
6148 }
6149 if ((Subtarget->hasFullFP16())) {
6150 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
6151 }
6152 return Register();
6153}
6154
6155Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
6156 if ((Subtarget->hasFPARMv8())) {
6157 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
6158 }
6159 return Register();
6160}
6161
6162Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
6163 if ((Subtarget->hasFPRCVT())) {
6164 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
6165 }
6166 if ((Subtarget->hasFPARMv8())) {
6167 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
6168 }
6169 return Register();
6170}
6171
6172Register fastEmit_ISD_UINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
6173switch (RetVT.SimpleTy) {
6174 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
6175 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
6176 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
6177 default: return Register();
6178}
6179}
6180
6181Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
6182 if ((Subtarget->hasFPRCVT())) {
6183 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
6184 }
6185 if ((Subtarget->hasFullFP16())) {
6186 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
6187 }
6188 return Register();
6189}
6190
6191Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
6192 if ((Subtarget->hasFPRCVT())) {
6193 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
6194 }
6195 if ((Subtarget->hasFPARMv8())) {
6196 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
6197 }
6198 return Register();
6199}
6200
6201Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
6202 if ((Subtarget->hasFPARMv8())) {
6203 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
6204 }
6205 return Register();
6206}
6207
6208Register fastEmit_ISD_UINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
6209switch (RetVT.SimpleTy) {
6210 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
6211 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
6212 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
6213 default: return Register();
6214}
6215}
6216
6217Register fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
6218 if (RetVT.SimpleTy != MVT::v4f16)
6219 return Register();
6220 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
6221 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
6222 }
6223 return Register();
6224}
6225
6226Register fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
6227 if (RetVT.SimpleTy != MVT::v8f16)
6228 return Register();
6229 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
6230 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
6231 }
6232 return Register();
6233}
6234
6235Register fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
6236 if (RetVT.SimpleTy != MVT::v2f32)
6237 return Register();
6238 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
6239 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
6240 }
6241 return Register();
6242}
6243
6244Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
6245 if (RetVT.SimpleTy != MVT::v4f32)
6246 return Register();
6247 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
6248 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
6249 }
6250 return Register();
6251}
6252
6253Register fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
6254 if (RetVT.SimpleTy != MVT::v2f64)
6255 return Register();
6256 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
6257 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
6258 }
6259 return Register();
6260}
6261
6262Register fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
6263 switch (VT.SimpleTy) {
6264 case MVT::i32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_r(RetVT, Op0);
6265 case MVT::i64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_r(RetVT, Op0);
6266 case MVT::v4i16: return fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
6267 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
6268 case MVT::v2i32: return fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
6269 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
6270 case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
6271 default: return Register();
6272 }
6273}
6274
6275// FastEmit functions for ISD::VECREDUCE_ADD.
6276
6277Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i8_r(MVT RetVT, Register Op0) {
6278 if (RetVT.SimpleTy != MVT::i8)
6279 return Register();
6280 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6281}
6282
6283Register fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(MVT RetVT, Register Op0) {
6284 if (RetVT.SimpleTy != MVT::i8)
6285 return Register();
6286 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6287}
6288
6289Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i16_r(MVT RetVT, Register Op0) {
6290 if (RetVT.SimpleTy != MVT::i16)
6291 return Register();
6292 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6293}
6294
6295Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(MVT RetVT, Register Op0) {
6296 if (RetVT.SimpleTy != MVT::i16)
6297 return Register();
6298 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6299}
6300
6301Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(MVT RetVT, Register Op0) {
6302 if (RetVT.SimpleTy != MVT::i32)
6303 return Register();
6304 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6305}
6306
6307Register fastEmit_ISD_VECREDUCE_ADD_MVT_v2i64_r(MVT RetVT, Register Op0) {
6308 if (RetVT.SimpleTy != MVT::i64)
6309 return Register();
6310 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6311}
6312
6313Register fastEmit_ISD_VECREDUCE_ADD_r(MVT VT, MVT RetVT, Register Op0) {
6314 switch (VT.SimpleTy) {
6315 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i8_r(RetVT, Op0);
6316 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(RetVT, Op0);
6317 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i16_r(RetVT, Op0);
6318 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(RetVT, Op0);
6319 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(RetVT, Op0);
6320 case MVT::v2i64: return fastEmit_ISD_VECREDUCE_ADD_MVT_v2i64_r(RetVT, Op0);
6321 default: return Register();
6322 }
6323}
6324
6325// FastEmit functions for ISD::VECREDUCE_FADD.
6326
6327Register fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(MVT RetVT, Register Op0) {
6328 if (RetVT.SimpleTy != MVT::f32)
6329 return Register();
6330 return fastEmitInst_r(MachineInstOpcode: AArch64::FADDPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6331}
6332
6333Register fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(MVT RetVT, Register Op0) {
6334 if (RetVT.SimpleTy != MVT::f64)
6335 return Register();
6336 return fastEmitInst_r(MachineInstOpcode: AArch64::FADDPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6337}
6338
6339Register fastEmit_ISD_VECREDUCE_FADD_r(MVT VT, MVT RetVT, Register Op0) {
6340 switch (VT.SimpleTy) {
6341 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(RetVT, Op0);
6342 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(RetVT, Op0);
6343 default: return Register();
6344 }
6345}
6346
6347// FastEmit functions for ISD::VECREDUCE_FMAX.
6348
6349Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f16_r(MVT RetVT, Register Op0) {
6350 if (RetVT.SimpleTy != MVT::f16)
6351 return Register();
6352 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6353 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6354 }
6355 return Register();
6356}
6357
6358Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v8f16_r(MVT RetVT, Register Op0) {
6359 if (RetVT.SimpleTy != MVT::f16)
6360 return Register();
6361 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6362 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6363 }
6364 return Register();
6365}
6366
6367Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f32_r(MVT RetVT, Register Op0) {
6368 if (RetVT.SimpleTy != MVT::f32)
6369 return Register();
6370 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6371}
6372
6373Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f32_r(MVT RetVT, Register Op0) {
6374 if (RetVT.SimpleTy != MVT::f32)
6375 return Register();
6376 if ((Subtarget->isNeonAvailable())) {
6377 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6378 }
6379 return Register();
6380}
6381
6382Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f64_r(MVT RetVT, Register Op0) {
6383 if (RetVT.SimpleTy != MVT::f64)
6384 return Register();
6385 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6386}
6387
6388Register fastEmit_ISD_VECREDUCE_FMAX_r(MVT VT, MVT RetVT, Register Op0) {
6389 switch (VT.SimpleTy) {
6390 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f16_r(RetVT, Op0);
6391 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v8f16_r(RetVT, Op0);
6392 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f32_r(RetVT, Op0);
6393 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f32_r(RetVT, Op0);
6394 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f64_r(RetVT, Op0);
6395 default: return Register();
6396 }
6397}
6398
6399// FastEmit functions for ISD::VECREDUCE_FMAXIMUM.
6400
6401Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f16_r(MVT RetVT, Register Op0) {
6402 if (RetVT.SimpleTy != MVT::f16)
6403 return Register();
6404 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6405 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6406 }
6407 return Register();
6408}
6409
6410Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v8f16_r(MVT RetVT, Register Op0) {
6411 if (RetVT.SimpleTy != MVT::f16)
6412 return Register();
6413 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6414 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6415 }
6416 return Register();
6417}
6418
6419Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f32_r(MVT RetVT, Register Op0) {
6420 if (RetVT.SimpleTy != MVT::f32)
6421 return Register();
6422 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6423}
6424
6425Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f32_r(MVT RetVT, Register Op0) {
6426 if (RetVT.SimpleTy != MVT::f32)
6427 return Register();
6428 if ((Subtarget->isNeonAvailable())) {
6429 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6430 }
6431 return Register();
6432}
6433
6434Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f64_r(MVT RetVT, Register Op0) {
6435 if (RetVT.SimpleTy != MVT::f64)
6436 return Register();
6437 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6438}
6439
6440Register fastEmit_ISD_VECREDUCE_FMAXIMUM_r(MVT VT, MVT RetVT, Register Op0) {
6441 switch (VT.SimpleTy) {
6442 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f16_r(RetVT, Op0);
6443 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v8f16_r(RetVT, Op0);
6444 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f32_r(RetVT, Op0);
6445 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f32_r(RetVT, Op0);
6446 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f64_r(RetVT, Op0);
6447 default: return Register();
6448 }
6449}
6450
6451// FastEmit functions for ISD::VECREDUCE_FMIN.
6452
6453Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f16_r(MVT RetVT, Register Op0) {
6454 if (RetVT.SimpleTy != MVT::f16)
6455 return Register();
6456 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6457 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6458 }
6459 return Register();
6460}
6461
6462Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v8f16_r(MVT RetVT, Register Op0) {
6463 if (RetVT.SimpleTy != MVT::f16)
6464 return Register();
6465 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6466 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6467 }
6468 return Register();
6469}
6470
6471Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f32_r(MVT RetVT, Register Op0) {
6472 if (RetVT.SimpleTy != MVT::f32)
6473 return Register();
6474 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6475}
6476
6477Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f32_r(MVT RetVT, Register Op0) {
6478 if (RetVT.SimpleTy != MVT::f32)
6479 return Register();
6480 if ((Subtarget->isNeonAvailable())) {
6481 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6482 }
6483 return Register();
6484}
6485
6486Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f64_r(MVT RetVT, Register Op0) {
6487 if (RetVT.SimpleTy != MVT::f64)
6488 return Register();
6489 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6490}
6491
6492Register fastEmit_ISD_VECREDUCE_FMIN_r(MVT VT, MVT RetVT, Register Op0) {
6493 switch (VT.SimpleTy) {
6494 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f16_r(RetVT, Op0);
6495 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v8f16_r(RetVT, Op0);
6496 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f32_r(RetVT, Op0);
6497 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f32_r(RetVT, Op0);
6498 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f64_r(RetVT, Op0);
6499 default: return Register();
6500 }
6501}
6502
6503// FastEmit functions for ISD::VECREDUCE_FMINIMUM.
6504
6505Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f16_r(MVT RetVT, Register Op0) {
6506 if (RetVT.SimpleTy != MVT::f16)
6507 return Register();
6508 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6509 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6510 }
6511 return Register();
6512}
6513
6514Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v8f16_r(MVT RetVT, Register Op0) {
6515 if (RetVT.SimpleTy != MVT::f16)
6516 return Register();
6517 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6518 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6519 }
6520 return Register();
6521}
6522
6523Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f32_r(MVT RetVT, Register Op0) {
6524 if (RetVT.SimpleTy != MVT::f32)
6525 return Register();
6526 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6527}
6528
6529Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f32_r(MVT RetVT, Register Op0) {
6530 if (RetVT.SimpleTy != MVT::f32)
6531 return Register();
6532 if ((Subtarget->isNeonAvailable())) {
6533 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6534 }
6535 return Register();
6536}
6537
6538Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f64_r(MVT RetVT, Register Op0) {
6539 if (RetVT.SimpleTy != MVT::f64)
6540 return Register();
6541 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6542}
6543
6544Register fastEmit_ISD_VECREDUCE_FMINIMUM_r(MVT VT, MVT RetVT, Register Op0) {
6545 switch (VT.SimpleTy) {
6546 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f16_r(RetVT, Op0);
6547 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v8f16_r(RetVT, Op0);
6548 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f32_r(RetVT, Op0);
6549 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f32_r(RetVT, Op0);
6550 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f64_r(RetVT, Op0);
6551 default: return Register();
6552 }
6553}
6554
6555// FastEmit functions for ISD::VECREDUCE_SMAX.
6556
6557Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i8_r(MVT RetVT, Register Op0) {
6558 if (RetVT.SimpleTy != MVT::i8)
6559 return Register();
6560 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6561}
6562
6563Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v16i8_r(MVT RetVT, Register Op0) {
6564 if (RetVT.SimpleTy != MVT::i8)
6565 return Register();
6566 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6567}
6568
6569Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i16_r(MVT RetVT, Register Op0) {
6570 if (RetVT.SimpleTy != MVT::i16)
6571 return Register();
6572 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6573}
6574
6575Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i16_r(MVT RetVT, Register Op0) {
6576 if (RetVT.SimpleTy != MVT::i16)
6577 return Register();
6578 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6579}
6580
6581Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i32_r(MVT RetVT, Register Op0) {
6582 if (RetVT.SimpleTy != MVT::i32)
6583 return Register();
6584 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6585}
6586
6587Register fastEmit_ISD_VECREDUCE_SMAX_r(MVT VT, MVT RetVT, Register Op0) {
6588 switch (VT.SimpleTy) {
6589 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i8_r(RetVT, Op0);
6590 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v16i8_r(RetVT, Op0);
6591 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i16_r(RetVT, Op0);
6592 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i16_r(RetVT, Op0);
6593 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i32_r(RetVT, Op0);
6594 default: return Register();
6595 }
6596}
6597
6598// FastEmit functions for ISD::VECREDUCE_SMIN.
6599
6600Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i8_r(MVT RetVT, Register Op0) {
6601 if (RetVT.SimpleTy != MVT::i8)
6602 return Register();
6603 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6604}
6605
6606Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v16i8_r(MVT RetVT, Register Op0) {
6607 if (RetVT.SimpleTy != MVT::i8)
6608 return Register();
6609 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6610}
6611
6612Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i16_r(MVT RetVT, Register Op0) {
6613 if (RetVT.SimpleTy != MVT::i16)
6614 return Register();
6615 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6616}
6617
6618Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i16_r(MVT RetVT, Register Op0) {
6619 if (RetVT.SimpleTy != MVT::i16)
6620 return Register();
6621 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6622}
6623
6624Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i32_r(MVT RetVT, Register Op0) {
6625 if (RetVT.SimpleTy != MVT::i32)
6626 return Register();
6627 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6628}
6629
6630Register fastEmit_ISD_VECREDUCE_SMIN_r(MVT VT, MVT RetVT, Register Op0) {
6631 switch (VT.SimpleTy) {
6632 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i8_r(RetVT, Op0);
6633 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v16i8_r(RetVT, Op0);
6634 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i16_r(RetVT, Op0);
6635 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i16_r(RetVT, Op0);
6636 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i32_r(RetVT, Op0);
6637 default: return Register();
6638 }
6639}
6640
6641// FastEmit functions for ISD::VECREDUCE_UMAX.
6642
6643Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i8_r(MVT RetVT, Register Op0) {
6644 if (RetVT.SimpleTy != MVT::i8)
6645 return Register();
6646 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6647}
6648
6649Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v16i8_r(MVT RetVT, Register Op0) {
6650 if (RetVT.SimpleTy != MVT::i8)
6651 return Register();
6652 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6653}
6654
6655Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i16_r(MVT RetVT, Register Op0) {
6656 if (RetVT.SimpleTy != MVT::i16)
6657 return Register();
6658 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6659}
6660
6661Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i16_r(MVT RetVT, Register Op0) {
6662 if (RetVT.SimpleTy != MVT::i16)
6663 return Register();
6664 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6665}
6666
6667Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i32_r(MVT RetVT, Register Op0) {
6668 if (RetVT.SimpleTy != MVT::i32)
6669 return Register();
6670 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6671}
6672
6673Register fastEmit_ISD_VECREDUCE_UMAX_r(MVT VT, MVT RetVT, Register Op0) {
6674 switch (VT.SimpleTy) {
6675 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i8_r(RetVT, Op0);
6676 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v16i8_r(RetVT, Op0);
6677 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i16_r(RetVT, Op0);
6678 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i16_r(RetVT, Op0);
6679 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i32_r(RetVT, Op0);
6680 default: return Register();
6681 }
6682}
6683
6684// FastEmit functions for ISD::VECREDUCE_UMIN.
6685
6686Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i8_r(MVT RetVT, Register Op0) {
6687 if (RetVT.SimpleTy != MVT::i8)
6688 return Register();
6689 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6690}
6691
6692Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v16i8_r(MVT RetVT, Register Op0) {
6693 if (RetVT.SimpleTy != MVT::i8)
6694 return Register();
6695 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6696}
6697
6698Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i16_r(MVT RetVT, Register Op0) {
6699 if (RetVT.SimpleTy != MVT::i16)
6700 return Register();
6701 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6702}
6703
6704Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i16_r(MVT RetVT, Register Op0) {
6705 if (RetVT.SimpleTy != MVT::i16)
6706 return Register();
6707 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6708}
6709
6710Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i32_r(MVT RetVT, Register Op0) {
6711 if (RetVT.SimpleTy != MVT::i32)
6712 return Register();
6713 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6714}
6715
6716Register fastEmit_ISD_VECREDUCE_UMIN_r(MVT VT, MVT RetVT, Register Op0) {
6717 switch (VT.SimpleTy) {
6718 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i8_r(RetVT, Op0);
6719 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v16i8_r(RetVT, Op0);
6720 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i16_r(RetVT, Op0);
6721 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i16_r(RetVT, Op0);
6722 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i32_r(RetVT, Op0);
6723 default: return Register();
6724 }
6725}
6726
6727// FastEmit functions for ISD::VECTOR_REVERSE.
6728
6729Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(MVT RetVT, Register Op0) {
6730 if (RetVT.SimpleTy != MVT::nxv2i1)
6731 return Register();
6732 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6733 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_D, RC: &AArch64::PPRRegClass, Op0);
6734 }
6735 return Register();
6736}
6737
6738Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(MVT RetVT, Register Op0) {
6739 if (RetVT.SimpleTy != MVT::nxv4i1)
6740 return Register();
6741 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6742 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_S, RC: &AArch64::PPRRegClass, Op0);
6743 }
6744 return Register();
6745}
6746
6747Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(MVT RetVT, Register Op0) {
6748 if (RetVT.SimpleTy != MVT::nxv8i1)
6749 return Register();
6750 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6751 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_H, RC: &AArch64::PPRRegClass, Op0);
6752 }
6753 return Register();
6754}
6755
6756Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(MVT RetVT, Register Op0) {
6757 if (RetVT.SimpleTy != MVT::nxv16i1)
6758 return Register();
6759 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6760 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_B, RC: &AArch64::PPRRegClass, Op0);
6761 }
6762 return Register();
6763}
6764
6765Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
6766 if (RetVT.SimpleTy != MVT::nxv16i8)
6767 return Register();
6768 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6769 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_B, RC: &AArch64::ZPRRegClass, Op0);
6770 }
6771 return Register();
6772}
6773
6774Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
6775 if (RetVT.SimpleTy != MVT::nxv8i16)
6776 return Register();
6777 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6778 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6779 }
6780 return Register();
6781}
6782
6783Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
6784 if (RetVT.SimpleTy != MVT::nxv4i32)
6785 return Register();
6786 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6787 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6788 }
6789 return Register();
6790}
6791
6792Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(MVT RetVT, Register Op0) {
6793 if (RetVT.SimpleTy != MVT::nxv2i64)
6794 return Register();
6795 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6796 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6797 }
6798 return Register();
6799}
6800
6801Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(MVT RetVT, Register Op0) {
6802 if (RetVT.SimpleTy != MVT::nxv2f16)
6803 return Register();
6804 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6805 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6806 }
6807 return Register();
6808}
6809
6810Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(MVT RetVT, Register Op0) {
6811 if (RetVT.SimpleTy != MVT::nxv4f16)
6812 return Register();
6813 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6814 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6815 }
6816 return Register();
6817}
6818
6819Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
6820 if (RetVT.SimpleTy != MVT::nxv8f16)
6821 return Register();
6822 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6823 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6824 }
6825 return Register();
6826}
6827
6828Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(MVT RetVT, Register Op0) {
6829 if (RetVT.SimpleTy != MVT::nxv2bf16)
6830 return Register();
6831 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6832 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6833 }
6834 return Register();
6835}
6836
6837Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(MVT RetVT, Register Op0) {
6838 if (RetVT.SimpleTy != MVT::nxv4bf16)
6839 return Register();
6840 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6841 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6842 }
6843 return Register();
6844}
6845
6846Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(MVT RetVT, Register Op0) {
6847 if (RetVT.SimpleTy != MVT::nxv8bf16)
6848 return Register();
6849 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6850 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6851 }
6852 return Register();
6853}
6854
6855Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(MVT RetVT, Register Op0) {
6856 if (RetVT.SimpleTy != MVT::nxv2f32)
6857 return Register();
6858 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6859 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6860 }
6861 return Register();
6862}
6863
6864Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
6865 if (RetVT.SimpleTy != MVT::nxv4f32)
6866 return Register();
6867 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6868 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6869 }
6870 return Register();
6871}
6872
6873Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
6874 if (RetVT.SimpleTy != MVT::nxv2f64)
6875 return Register();
6876 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6877 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6878 }
6879 return Register();
6880}
6881
6882Register fastEmit_ISD_VECTOR_REVERSE_r(MVT VT, MVT RetVT, Register Op0) {
6883 switch (VT.SimpleTy) {
6884 case MVT::nxv2i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(RetVT, Op0);
6885 case MVT::nxv4i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(RetVT, Op0);
6886 case MVT::nxv8i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(RetVT, Op0);
6887 case MVT::nxv16i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(RetVT, Op0);
6888 case MVT::nxv16i8: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(RetVT, Op0);
6889 case MVT::nxv8i16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(RetVT, Op0);
6890 case MVT::nxv4i32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(RetVT, Op0);
6891 case MVT::nxv2i64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(RetVT, Op0);
6892 case MVT::nxv2f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(RetVT, Op0);
6893 case MVT::nxv4f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(RetVT, Op0);
6894 case MVT::nxv8f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(RetVT, Op0);
6895 case MVT::nxv2bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(RetVT, Op0);
6896 case MVT::nxv4bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(RetVT, Op0);
6897 case MVT::nxv8bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(RetVT, Op0);
6898 case MVT::nxv2f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(RetVT, Op0);
6899 case MVT::nxv4f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(RetVT, Op0);
6900 case MVT::nxv2f64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(RetVT, Op0);
6901 default: return Register();
6902 }
6903}
6904
6905// Top-level FastEmit function.
6906
6907Register fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, Register Op0) override {
6908 switch (Opcode) {
6909 case AArch64ISD::ALLOCATE_ZA_BUFFER: return fastEmit_AArch64ISD_ALLOCATE_ZA_BUFFER_r(VT, RetVT, Op0);
6910 case AArch64ISD::ALLOC_SME_SAVE_BUFFER: return fastEmit_AArch64ISD_ALLOC_SME_SAVE_BUFFER_r(VT, RetVT, Op0);
6911 case AArch64ISD::CALL: return fastEmit_AArch64ISD_CALL_r(VT, RetVT, Op0);
6912 case AArch64ISD::COALESCER_BARRIER: return fastEmit_AArch64ISD_COALESCER_BARRIER_r(VT, RetVT, Op0);
6913 case AArch64ISD::DUP: return fastEmit_AArch64ISD_DUP_r(VT, RetVT, Op0);
6914 case AArch64ISD::FCVTXN: return fastEmit_AArch64ISD_FCVTXN_r(VT, RetVT, Op0);
6915 case AArch64ISD::FRECPE: return fastEmit_AArch64ISD_FRECPE_r(VT, RetVT, Op0);
6916 case AArch64ISD::FRSQRTE: return fastEmit_AArch64ISD_FRSQRTE_r(VT, RetVT, Op0);
6917 case AArch64ISD::PROBED_ALLOCA: return fastEmit_AArch64ISD_PROBED_ALLOCA_r(VT, RetVT, Op0);
6918 case AArch64ISD::REV16: return fastEmit_AArch64ISD_REV16_r(VT, RetVT, Op0);
6919 case AArch64ISD::REV32: return fastEmit_AArch64ISD_REV32_r(VT, RetVT, Op0);
6920 case AArch64ISD::REV64: return fastEmit_AArch64ISD_REV64_r(VT, RetVT, Op0);
6921 case AArch64ISD::SADDLP: return fastEmit_AArch64ISD_SADDLP_r(VT, RetVT, Op0);
6922 case AArch64ISD::SITOF: return fastEmit_AArch64ISD_SITOF_r(VT, RetVT, Op0);
6923 case AArch64ISD::SQABS: return fastEmit_AArch64ISD_SQABS_r(VT, RetVT, Op0);
6924 case AArch64ISD::SQNEG: return fastEmit_AArch64ISD_SQNEG_r(VT, RetVT, Op0);
6925 case AArch64ISD::SUNPKHI: return fastEmit_AArch64ISD_SUNPKHI_r(VT, RetVT, Op0);
6926 case AArch64ISD::SUNPKLO: return fastEmit_AArch64ISD_SUNPKLO_r(VT, RetVT, Op0);
6927 case AArch64ISD::UADDLP: return fastEmit_AArch64ISD_UADDLP_r(VT, RetVT, Op0);
6928 case AArch64ISD::UITOF: return fastEmit_AArch64ISD_UITOF_r(VT, RetVT, Op0);
6929 case AArch64ISD::UUNPKHI: return fastEmit_AArch64ISD_UUNPKHI_r(VT, RetVT, Op0);
6930 case AArch64ISD::UUNPKLO: return fastEmit_AArch64ISD_UUNPKLO_r(VT, RetVT, Op0);
6931 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
6932 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
6933 case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0);
6934 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0);
6935 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
6936 case ISD::CTLS: return fastEmit_ISD_CTLS_r(VT, RetVT, Op0);
6937 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
6938 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
6939 case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0);
6940 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
6941 case ISD::FCEIL: return fastEmit_ISD_FCEIL_r(VT, RetVT, Op0);
6942 case ISD::FFLOOR: return fastEmit_ISD_FFLOOR_r(VT, RetVT, Op0);
6943 case ISD::FNEARBYINT: return fastEmit_ISD_FNEARBYINT_r(VT, RetVT, Op0);
6944 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
6945 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
6946 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
6947 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
6948 case ISD::FP_TO_SINT_SAT: return fastEmit_ISD_FP_TO_SINT_SAT_r(VT, RetVT, Op0);
6949 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
6950 case ISD::FP_TO_UINT_SAT: return fastEmit_ISD_FP_TO_UINT_SAT_r(VT, RetVT, Op0);
6951 case ISD::FRINT: return fastEmit_ISD_FRINT_r(VT, RetVT, Op0);
6952 case ISD::FROUND: return fastEmit_ISD_FROUND_r(VT, RetVT, Op0);
6953 case ISD::FROUNDEVEN: return fastEmit_ISD_FROUNDEVEN_r(VT, RetVT, Op0);
6954 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
6955 case ISD::FTRUNC: return fastEmit_ISD_FTRUNC_r(VT, RetVT, Op0);
6956 case ISD::LLROUND: return fastEmit_ISD_LLROUND_r(VT, RetVT, Op0);
6957 case ISD::LROUND: return fastEmit_ISD_LROUND_r(VT, RetVT, Op0);
6958 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
6959 case ISD::SPLAT_VECTOR: return fastEmit_ISD_SPLAT_VECTOR_r(VT, RetVT, Op0);
6960 case ISD::STRICT_FCEIL: return fastEmit_ISD_STRICT_FCEIL_r(VT, RetVT, Op0);
6961 case ISD::STRICT_FFLOOR: return fastEmit_ISD_STRICT_FFLOOR_r(VT, RetVT, Op0);
6962 case ISD::STRICT_FNEARBYINT: return fastEmit_ISD_STRICT_FNEARBYINT_r(VT, RetVT, Op0);
6963 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0);
6964 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0);
6965 case ISD::STRICT_FP_TO_SINT: return fastEmit_ISD_STRICT_FP_TO_SINT_r(VT, RetVT, Op0);
6966 case ISD::STRICT_FP_TO_UINT: return fastEmit_ISD_STRICT_FP_TO_UINT_r(VT, RetVT, Op0);
6967 case ISD::STRICT_FRINT: return fastEmit_ISD_STRICT_FRINT_r(VT, RetVT, Op0);
6968 case ISD::STRICT_FROUND: return fastEmit_ISD_STRICT_FROUND_r(VT, RetVT, Op0);
6969 case ISD::STRICT_FROUNDEVEN: return fastEmit_ISD_STRICT_FROUNDEVEN_r(VT, RetVT, Op0);
6970 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0);
6971 case ISD::STRICT_FTRUNC: return fastEmit_ISD_STRICT_FTRUNC_r(VT, RetVT, Op0);
6972 case ISD::STRICT_LLROUND: return fastEmit_ISD_STRICT_LLROUND_r(VT, RetVT, Op0);
6973 case ISD::STRICT_LROUND: return fastEmit_ISD_STRICT_LROUND_r(VT, RetVT, Op0);
6974 case ISD::STRICT_SINT_TO_FP: return fastEmit_ISD_STRICT_SINT_TO_FP_r(VT, RetVT, Op0);
6975 case ISD::STRICT_UINT_TO_FP: return fastEmit_ISD_STRICT_UINT_TO_FP_r(VT, RetVT, Op0);
6976 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
6977 case ISD::TRUNCATE_SSAT_S: return fastEmit_ISD_TRUNCATE_SSAT_S_r(VT, RetVT, Op0);
6978 case ISD::TRUNCATE_SSAT_U: return fastEmit_ISD_TRUNCATE_SSAT_U_r(VT, RetVT, Op0);
6979 case ISD::TRUNCATE_USAT_U: return fastEmit_ISD_TRUNCATE_USAT_U_r(VT, RetVT, Op0);
6980 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
6981 case ISD::VECREDUCE_ADD: return fastEmit_ISD_VECREDUCE_ADD_r(VT, RetVT, Op0);
6982 case ISD::VECREDUCE_FADD: return fastEmit_ISD_VECREDUCE_FADD_r(VT, RetVT, Op0);
6983 case ISD::VECREDUCE_FMAX: return fastEmit_ISD_VECREDUCE_FMAX_r(VT, RetVT, Op0);
6984 case ISD::VECREDUCE_FMAXIMUM: return fastEmit_ISD_VECREDUCE_FMAXIMUM_r(VT, RetVT, Op0);
6985 case ISD::VECREDUCE_FMIN: return fastEmit_ISD_VECREDUCE_FMIN_r(VT, RetVT, Op0);
6986 case ISD::VECREDUCE_FMINIMUM: return fastEmit_ISD_VECREDUCE_FMINIMUM_r(VT, RetVT, Op0);
6987 case ISD::VECREDUCE_SMAX: return fastEmit_ISD_VECREDUCE_SMAX_r(VT, RetVT, Op0);
6988 case ISD::VECREDUCE_SMIN: return fastEmit_ISD_VECREDUCE_SMIN_r(VT, RetVT, Op0);
6989 case ISD::VECREDUCE_UMAX: return fastEmit_ISD_VECREDUCE_UMAX_r(VT, RetVT, Op0);
6990 case ISD::VECREDUCE_UMIN: return fastEmit_ISD_VECREDUCE_UMIN_r(VT, RetVT, Op0);
6991 case ISD::VECTOR_REVERSE: return fastEmit_ISD_VECTOR_REVERSE_r(VT, RetVT, Op0);
6992 default: return Register();
6993 }
6994}
6995
6996// FastEmit functions for AArch64ISD::ADDP.
6997
6998Register fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6999 if (RetVT.SimpleTy != MVT::v8i8)
7000 return Register();
7001 if ((Subtarget->isNeonAvailable())) {
7002 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
7003 }
7004 return Register();
7005}
7006
7007Register fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7008 if (RetVT.SimpleTy != MVT::v16i8)
7009 return Register();
7010 if ((Subtarget->isNeonAvailable())) {
7011 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
7012 }
7013 return Register();
7014}
7015
7016Register fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7017 if (RetVT.SimpleTy != MVT::v4i16)
7018 return Register();
7019 if ((Subtarget->isNeonAvailable())) {
7020 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7021 }
7022 return Register();
7023}
7024
7025Register fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7026 if (RetVT.SimpleTy != MVT::v8i16)
7027 return Register();
7028 if ((Subtarget->isNeonAvailable())) {
7029 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7030 }
7031 return Register();
7032}
7033
7034Register fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7035 if (RetVT.SimpleTy != MVT::v2i32)
7036 return Register();
7037 if ((Subtarget->isNeonAvailable())) {
7038 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7039 }
7040 return Register();
7041}
7042
7043Register fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7044 if (RetVT.SimpleTy != MVT::v4i32)
7045 return Register();
7046 if ((Subtarget->isNeonAvailable())) {
7047 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7048 }
7049 return Register();
7050}
7051
7052Register fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7053 if (RetVT.SimpleTy != MVT::v2i64)
7054 return Register();
7055 if ((Subtarget->isNeonAvailable())) {
7056 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7057 }
7058 return Register();
7059}
7060
7061Register fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7062 if (RetVT.SimpleTy != MVT::v4f16)
7063 return Register();
7064 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7065 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7066 }
7067 return Register();
7068}
7069
7070Register fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7071 if (RetVT.SimpleTy != MVT::v8f16)
7072 return Register();
7073 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7074 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7075 }
7076 return Register();
7077}
7078
7079Register fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7080 if (RetVT.SimpleTy != MVT::v2f32)
7081 return Register();
7082 if ((Subtarget->isNeonAvailable())) {
7083 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7084 }
7085 return Register();
7086}
7087
7088Register fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7089 if (RetVT.SimpleTy != MVT::v4f32)
7090 return Register();
7091 if ((Subtarget->isNeonAvailable())) {
7092 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7093 }
7094 return Register();
7095}
7096
7097Register fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7098 if (RetVT.SimpleTy != MVT::v2f64)
7099 return Register();
7100 if ((Subtarget->isNeonAvailable())) {
7101 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7102 }
7103 return Register();
7104}
7105
7106Register fastEmit_AArch64ISD_ADDP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7107 switch (VT.SimpleTy) {
7108 case MVT::v8i8: return fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(RetVT, Op0, Op1);
7109 case MVT::v16i8: return fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(RetVT, Op0, Op1);
7110 case MVT::v4i16: return fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(RetVT, Op0, Op1);
7111 case MVT::v8i16: return fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(RetVT, Op0, Op1);
7112 case MVT::v2i32: return fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(RetVT, Op0, Op1);
7113 case MVT::v4i32: return fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(RetVT, Op0, Op1);
7114 case MVT::v2i64: return fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(RetVT, Op0, Op1);
7115 case MVT::v4f16: return fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(RetVT, Op0, Op1);
7116 case MVT::v8f16: return fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(RetVT, Op0, Op1);
7117 case MVT::v2f32: return fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(RetVT, Op0, Op1);
7118 case MVT::v4f32: return fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(RetVT, Op0, Op1);
7119 case MVT::v2f64: return fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(RetVT, Op0, Op1);
7120 default: return Register();
7121 }
7122}
7123
7124// FastEmit functions for AArch64ISD::BIC.
7125
7126Register fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7127 if (RetVT.SimpleTy != MVT::nxv16i8)
7128 return Register();
7129 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7130 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
7131 }
7132 return Register();
7133}
7134
7135Register fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7136 if (RetVT.SimpleTy != MVT::nxv8i16)
7137 return Register();
7138 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7139 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
7140 }
7141 return Register();
7142}
7143
7144Register fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7145 if (RetVT.SimpleTy != MVT::nxv4i32)
7146 return Register();
7147 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7148 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
7149 }
7150 return Register();
7151}
7152
7153Register fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7154 if (RetVT.SimpleTy != MVT::nxv2i64)
7155 return Register();
7156 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7157 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
7158 }
7159 return Register();
7160}
7161
7162Register fastEmit_AArch64ISD_BIC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7163 switch (VT.SimpleTy) {
7164 case MVT::nxv16i8: return fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(RetVT, Op0, Op1);
7165 case MVT::nxv8i16: return fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(RetVT, Op0, Op1);
7166 case MVT::nxv4i32: return fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(RetVT, Op0, Op1);
7167 case MVT::nxv2i64: return fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(RetVT, Op0, Op1);
7168 default: return Register();
7169 }
7170}
7171
7172// FastEmit functions for AArch64ISD::FCMEQ.
7173
7174Register fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7175 if (RetVT.SimpleTy != MVT::i32)
7176 return Register();
7177 if ((Subtarget->isNeonAvailable())) {
7178 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7179 }
7180 return Register();
7181}
7182
7183Register fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7184 if (RetVT.SimpleTy != MVT::i64)
7185 return Register();
7186 if ((Subtarget->isNeonAvailable())) {
7187 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7188 }
7189 return Register();
7190}
7191
7192Register fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7193 if (RetVT.SimpleTy != MVT::v4i16)
7194 return Register();
7195 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7196 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7197 }
7198 return Register();
7199}
7200
7201Register fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7202 if (RetVT.SimpleTy != MVT::v8i16)
7203 return Register();
7204 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7205 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7206 }
7207 return Register();
7208}
7209
7210Register fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7211 if (RetVT.SimpleTy != MVT::v2i32)
7212 return Register();
7213 if ((Subtarget->isNeonAvailable())) {
7214 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7215 }
7216 return Register();
7217}
7218
7219Register fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7220 if (RetVT.SimpleTy != MVT::v4i32)
7221 return Register();
7222 if ((Subtarget->isNeonAvailable())) {
7223 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7224 }
7225 return Register();
7226}
7227
7228Register fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7229 if (RetVT.SimpleTy != MVT::v1i64)
7230 return Register();
7231 if ((Subtarget->isNeonAvailable())) {
7232 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7233 }
7234 return Register();
7235}
7236
7237Register fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7238 if (RetVT.SimpleTy != MVT::v2i64)
7239 return Register();
7240 if ((Subtarget->isNeonAvailable())) {
7241 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7242 }
7243 return Register();
7244}
7245
7246Register fastEmit_AArch64ISD_FCMEQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7247 switch (VT.SimpleTy) {
7248 case MVT::f32: return fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(RetVT, Op0, Op1);
7249 case MVT::f64: return fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(RetVT, Op0, Op1);
7250 case MVT::v4f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(RetVT, Op0, Op1);
7251 case MVT::v8f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(RetVT, Op0, Op1);
7252 case MVT::v2f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(RetVT, Op0, Op1);
7253 case MVT::v4f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(RetVT, Op0, Op1);
7254 case MVT::v1f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(RetVT, Op0, Op1);
7255 case MVT::v2f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(RetVT, Op0, Op1);
7256 default: return Register();
7257 }
7258}
7259
7260// FastEmit functions for AArch64ISD::FCMGE.
7261
7262Register fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7263 if (RetVT.SimpleTy != MVT::i32)
7264 return Register();
7265 if ((Subtarget->isNeonAvailable())) {
7266 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7267 }
7268 return Register();
7269}
7270
7271Register fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7272 if (RetVT.SimpleTy != MVT::i64)
7273 return Register();
7274 if ((Subtarget->isNeonAvailable())) {
7275 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7276 }
7277 return Register();
7278}
7279
7280Register fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7281 if (RetVT.SimpleTy != MVT::v4i16)
7282 return Register();
7283 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7284 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7285 }
7286 return Register();
7287}
7288
7289Register fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7290 if (RetVT.SimpleTy != MVT::v8i16)
7291 return Register();
7292 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7293 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7294 }
7295 return Register();
7296}
7297
7298Register fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7299 if (RetVT.SimpleTy != MVT::v2i32)
7300 return Register();
7301 if ((Subtarget->isNeonAvailable())) {
7302 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7303 }
7304 return Register();
7305}
7306
7307Register fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7308 if (RetVT.SimpleTy != MVT::v4i32)
7309 return Register();
7310 if ((Subtarget->isNeonAvailable())) {
7311 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7312 }
7313 return Register();
7314}
7315
7316Register fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7317 if (RetVT.SimpleTy != MVT::v1i64)
7318 return Register();
7319 if ((Subtarget->isNeonAvailable())) {
7320 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7321 }
7322 return Register();
7323}
7324
7325Register fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7326 if (RetVT.SimpleTy != MVT::v2i64)
7327 return Register();
7328 if ((Subtarget->isNeonAvailable())) {
7329 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7330 }
7331 return Register();
7332}
7333
7334Register fastEmit_AArch64ISD_FCMGE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7335 switch (VT.SimpleTy) {
7336 case MVT::f32: return fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(RetVT, Op0, Op1);
7337 case MVT::f64: return fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(RetVT, Op0, Op1);
7338 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(RetVT, Op0, Op1);
7339 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(RetVT, Op0, Op1);
7340 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(RetVT, Op0, Op1);
7341 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(RetVT, Op0, Op1);
7342 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(RetVT, Op0, Op1);
7343 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(RetVT, Op0, Op1);
7344 default: return Register();
7345 }
7346}
7347
7348// FastEmit functions for AArch64ISD::FCMGT.
7349
7350Register fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7351 if (RetVT.SimpleTy != MVT::i32)
7352 return Register();
7353 if ((Subtarget->isNeonAvailable())) {
7354 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7355 }
7356 return Register();
7357}
7358
7359Register fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7360 if (RetVT.SimpleTy != MVT::i64)
7361 return Register();
7362 if ((Subtarget->isNeonAvailable())) {
7363 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7364 }
7365 return Register();
7366}
7367
7368Register fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7369 if (RetVT.SimpleTy != MVT::v4i16)
7370 return Register();
7371 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7372 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7373 }
7374 return Register();
7375}
7376
7377Register fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7378 if (RetVT.SimpleTy != MVT::v8i16)
7379 return Register();
7380 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7381 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7382 }
7383 return Register();
7384}
7385
7386Register fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7387 if (RetVT.SimpleTy != MVT::v2i32)
7388 return Register();
7389 if ((Subtarget->isNeonAvailable())) {
7390 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7391 }
7392 return Register();
7393}
7394
7395Register fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7396 if (RetVT.SimpleTy != MVT::v4i32)
7397 return Register();
7398 if ((Subtarget->isNeonAvailable())) {
7399 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7400 }
7401 return Register();
7402}
7403
7404Register fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7405 if (RetVT.SimpleTy != MVT::v1i64)
7406 return Register();
7407 if ((Subtarget->isNeonAvailable())) {
7408 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7409 }
7410 return Register();
7411}
7412
7413Register fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7414 if (RetVT.SimpleTy != MVT::v2i64)
7415 return Register();
7416 if ((Subtarget->isNeonAvailable())) {
7417 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7418 }
7419 return Register();
7420}
7421
7422Register fastEmit_AArch64ISD_FCMGT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7423 switch (VT.SimpleTy) {
7424 case MVT::f32: return fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(RetVT, Op0, Op1);
7425 case MVT::f64: return fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(RetVT, Op0, Op1);
7426 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(RetVT, Op0, Op1);
7427 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(RetVT, Op0, Op1);
7428 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(RetVT, Op0, Op1);
7429 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(RetVT, Op0, Op1);
7430 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(RetVT, Op0, Op1);
7431 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(RetVT, Op0, Op1);
7432 default: return Register();
7433 }
7434}
7435
7436// FastEmit functions for AArch64ISD::FCMP.
7437
7438Register fastEmit_AArch64ISD_FCMP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
7439 if (RetVT.SimpleTy != MVT::i32)
7440 return Register();
7441 if ((Subtarget->hasFullFP16())) {
7442 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
7443 }
7444 return Register();
7445}
7446
7447Register fastEmit_AArch64ISD_FCMP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7448 if (RetVT.SimpleTy != MVT::i32)
7449 return Register();
7450 if ((Subtarget->hasFPARMv8())) {
7451 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
7452 }
7453 return Register();
7454}
7455
7456Register fastEmit_AArch64ISD_FCMP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7457 if (RetVT.SimpleTy != MVT::i32)
7458 return Register();
7459 if ((Subtarget->hasFPARMv8())) {
7460 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
7461 }
7462 return Register();
7463}
7464
7465Register fastEmit_AArch64ISD_FCMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7466 switch (VT.SimpleTy) {
7467 case MVT::f16: return fastEmit_AArch64ISD_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
7468 case MVT::f32: return fastEmit_AArch64ISD_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
7469 case MVT::f64: return fastEmit_AArch64ISD_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
7470 default: return Register();
7471 }
7472}
7473
7474// FastEmit functions for AArch64ISD::FRECPS.
7475
7476Register fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7477 if (RetVT.SimpleTy != MVT::f32)
7478 return Register();
7479 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7480}
7481
7482Register fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7483 if (RetVT.SimpleTy != MVT::f64)
7484 return Register();
7485 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7486}
7487
7488Register fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7489 if (RetVT.SimpleTy != MVT::v2f32)
7490 return Register();
7491 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7492}
7493
7494Register fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7495 if (RetVT.SimpleTy != MVT::v4f32)
7496 return Register();
7497 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7498}
7499
7500Register fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7501 if (RetVT.SimpleTy != MVT::v2f64)
7502 return Register();
7503 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7504}
7505
7506Register fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7507 if (RetVT.SimpleTy != MVT::nxv8f16)
7508 return Register();
7509 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7510 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7511 }
7512 return Register();
7513}
7514
7515Register fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7516 if (RetVT.SimpleTy != MVT::nxv4f32)
7517 return Register();
7518 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7519 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7520 }
7521 return Register();
7522}
7523
7524Register fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7525 if (RetVT.SimpleTy != MVT::nxv2f64)
7526 return Register();
7527 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7528 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7529 }
7530 return Register();
7531}
7532
7533Register fastEmit_AArch64ISD_FRECPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7534 switch (VT.SimpleTy) {
7535 case MVT::f32: return fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(RetVT, Op0, Op1);
7536 case MVT::f64: return fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(RetVT, Op0, Op1);
7537 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(RetVT, Op0, Op1);
7538 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(RetVT, Op0, Op1);
7539 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(RetVT, Op0, Op1);
7540 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(RetVT, Op0, Op1);
7541 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(RetVT, Op0, Op1);
7542 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(RetVT, Op0, Op1);
7543 default: return Register();
7544 }
7545}
7546
7547// FastEmit functions for AArch64ISD::FRSQRTS.
7548
7549Register fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7550 if (RetVT.SimpleTy != MVT::f32)
7551 return Register();
7552 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7553}
7554
7555Register fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7556 if (RetVT.SimpleTy != MVT::f64)
7557 return Register();
7558 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7559}
7560
7561Register fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7562 if (RetVT.SimpleTy != MVT::v2f32)
7563 return Register();
7564 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7565}
7566
7567Register fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7568 if (RetVT.SimpleTy != MVT::v4f32)
7569 return Register();
7570 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7571}
7572
7573Register fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7574 if (RetVT.SimpleTy != MVT::v2f64)
7575 return Register();
7576 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7577}
7578
7579Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7580 if (RetVT.SimpleTy != MVT::nxv8f16)
7581 return Register();
7582 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7583 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7584 }
7585 return Register();
7586}
7587
7588Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7589 if (RetVT.SimpleTy != MVT::nxv4f32)
7590 return Register();
7591 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7592 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7593 }
7594 return Register();
7595}
7596
7597Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7598 if (RetVT.SimpleTy != MVT::nxv2f64)
7599 return Register();
7600 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7601 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7602 }
7603 return Register();
7604}
7605
7606Register fastEmit_AArch64ISD_FRSQRTS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7607 switch (VT.SimpleTy) {
7608 case MVT::f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(RetVT, Op0, Op1);
7609 case MVT::f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(RetVT, Op0, Op1);
7610 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(RetVT, Op0, Op1);
7611 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(RetVT, Op0, Op1);
7612 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(RetVT, Op0, Op1);
7613 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(RetVT, Op0, Op1);
7614 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(RetVT, Op0, Op1);
7615 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(RetVT, Op0, Op1);
7616 default: return Register();
7617 }
7618}
7619
7620// FastEmit functions for AArch64ISD::INIT_TPIDR2OBJ.
7621
7622Register fastEmit_AArch64ISD_INIT_TPIDR2OBJ_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
7623 if (RetVT.SimpleTy != MVT::isVoid)
7624 return Register();
7625 return fastEmitInst_rr(MachineInstOpcode: AArch64::InitTPIDR2Obj, RC: &AArch64::GPR64RegClass, Op0, Op1);
7626}
7627
7628Register fastEmit_AArch64ISD_INIT_TPIDR2OBJ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7629 switch (VT.SimpleTy) {
7630 case MVT::i64: return fastEmit_AArch64ISD_INIT_TPIDR2OBJ_MVT_i64_rr(RetVT, Op0, Op1);
7631 default: return Register();
7632 }
7633}
7634
7635// FastEmit functions for AArch64ISD::PMULL.
7636
7637Register fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7638 if (RetVT.SimpleTy != MVT::v8i16)
7639 return Register();
7640 if ((Subtarget->isNeonAvailable())) {
7641 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULLv8i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
7642 }
7643 return Register();
7644}
7645
7646Register fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7647 if (RetVT.SimpleTy != MVT::v16i8)
7648 return Register();
7649 if ((Subtarget->hasAES())) {
7650 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULLv1i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7651 }
7652 return Register();
7653}
7654
7655Register fastEmit_AArch64ISD_PMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7656 switch (VT.SimpleTy) {
7657 case MVT::v8i8: return fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
7658 case MVT::v1i64: return fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(RetVT, Op0, Op1);
7659 default: return Register();
7660 }
7661}
7662
7663// FastEmit functions for AArch64ISD::PTEST.
7664
7665Register fastEmit_AArch64ISD_PTEST_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7666 if (RetVT.SimpleTy != MVT::i32)
7667 return Register();
7668 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7669 return fastEmitInst_rr(MachineInstOpcode: AArch64::PTEST_PP, RC: &AArch64::PPRRegClass, Op0, Op1);
7670 }
7671 return Register();
7672}
7673
7674Register fastEmit_AArch64ISD_PTEST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7675 switch (VT.SimpleTy) {
7676 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7677 default: return Register();
7678 }
7679}
7680
7681// FastEmit functions for AArch64ISD::PTEST_ANY.
7682
7683Register fastEmit_AArch64ISD_PTEST_ANY_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7684 if (RetVT.SimpleTy != MVT::i32)
7685 return Register();
7686 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7687 return fastEmitInst_rr(MachineInstOpcode: AArch64::PTEST_PP_ANY, RC: &AArch64::PPRRegClass, Op0, Op1);
7688 }
7689 return Register();
7690}
7691
7692Register fastEmit_AArch64ISD_PTEST_ANY_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7693 switch (VT.SimpleTy) {
7694 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_ANY_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7695 default: return Register();
7696 }
7697}
7698
7699// FastEmit functions for AArch64ISD::PTEST_FIRST.
7700
7701Register fastEmit_AArch64ISD_PTEST_FIRST_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7702 if (RetVT.SimpleTy != MVT::i32)
7703 return Register();
7704 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7705 return fastEmitInst_rr(MachineInstOpcode: AArch64::PTEST_PP_FIRST, RC: &AArch64::PPRRegClass, Op0, Op1);
7706 }
7707 return Register();
7708}
7709
7710Register fastEmit_AArch64ISD_PTEST_FIRST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7711 switch (VT.SimpleTy) {
7712 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_FIRST_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7713 default: return Register();
7714 }
7715}
7716
7717// FastEmit functions for AArch64ISD::SMULL.
7718
7719Register fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7720 if (RetVT.SimpleTy != MVT::v8i16)
7721 return Register();
7722 if ((Subtarget->isNeonAvailable())) {
7723 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv8i8_v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7724 }
7725 return Register();
7726}
7727
7728Register fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7729 if (RetVT.SimpleTy != MVT::v4i32)
7730 return Register();
7731 if ((Subtarget->isNeonAvailable())) {
7732 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv4i16_v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7733 }
7734 return Register();
7735}
7736
7737Register fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7738 if (RetVT.SimpleTy != MVT::v2i64)
7739 return Register();
7740 if ((Subtarget->isNeonAvailable())) {
7741 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv2i32_v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7742 }
7743 return Register();
7744}
7745
7746Register fastEmit_AArch64ISD_SMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7747 switch (VT.SimpleTy) {
7748 case MVT::v8i8: return fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
7749 case MVT::v4i16: return fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(RetVT, Op0, Op1);
7750 case MVT::v2i32: return fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(RetVT, Op0, Op1);
7751 default: return Register();
7752 }
7753}
7754
7755// FastEmit functions for AArch64ISD::SQADD.
7756
7757Register fastEmit_AArch64ISD_SQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7758 if (RetVT.SimpleTy != MVT::f32)
7759 return Register();
7760 if ((Subtarget->isNeonAvailable())) {
7761 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7762 }
7763 return Register();
7764}
7765
7766Register fastEmit_AArch64ISD_SQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7767 if (RetVT.SimpleTy != MVT::f64)
7768 return Register();
7769 if ((Subtarget->isNeonAvailable())) {
7770 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7771 }
7772 return Register();
7773}
7774
7775Register fastEmit_AArch64ISD_SQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7776 switch (VT.SimpleTy) {
7777 case MVT::f32: return fastEmit_AArch64ISD_SQADD_MVT_f32_rr(RetVT, Op0, Op1);
7778 case MVT::f64: return fastEmit_AArch64ISD_SQADD_MVT_f64_rr(RetVT, Op0, Op1);
7779 default: return Register();
7780 }
7781}
7782
7783// FastEmit functions for AArch64ISD::SQDMULH.
7784
7785Register fastEmit_AArch64ISD_SQDMULH_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7786 if (RetVT.SimpleTy != MVT::f32)
7787 return Register();
7788 if ((Subtarget->isNeonAvailable())) {
7789 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7790 }
7791 return Register();
7792}
7793
7794Register fastEmit_AArch64ISD_SQDMULH_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7795 if (RetVT.SimpleTy != MVT::v4i16)
7796 return Register();
7797 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7798}
7799
7800Register fastEmit_AArch64ISD_SQDMULH_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7801 if (RetVT.SimpleTy != MVT::v8i16)
7802 return Register();
7803 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7804}
7805
7806Register fastEmit_AArch64ISD_SQDMULH_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7807 if (RetVT.SimpleTy != MVT::v2i32)
7808 return Register();
7809 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7810}
7811
7812Register fastEmit_AArch64ISD_SQDMULH_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7813 if (RetVT.SimpleTy != MVT::v4i32)
7814 return Register();
7815 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7816}
7817
7818Register fastEmit_AArch64ISD_SQDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7819 switch (VT.SimpleTy) {
7820 case MVT::f32: return fastEmit_AArch64ISD_SQDMULH_MVT_f32_rr(RetVT, Op0, Op1);
7821 case MVT::v4i16: return fastEmit_AArch64ISD_SQDMULH_MVT_v4i16_rr(RetVT, Op0, Op1);
7822 case MVT::v8i16: return fastEmit_AArch64ISD_SQDMULH_MVT_v8i16_rr(RetVT, Op0, Op1);
7823 case MVT::v2i32: return fastEmit_AArch64ISD_SQDMULH_MVT_v2i32_rr(RetVT, Op0, Op1);
7824 case MVT::v4i32: return fastEmit_AArch64ISD_SQDMULH_MVT_v4i32_rr(RetVT, Op0, Op1);
7825 default: return Register();
7826 }
7827}
7828
7829// FastEmit functions for AArch64ISD::SQDMULL.
7830
7831Register fastEmit_AArch64ISD_SQDMULL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7832 if (RetVT.SimpleTy != MVT::f64)
7833 return Register();
7834 if ((Subtarget->isNeonAvailable())) {
7835 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULLi32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7836 }
7837 return Register();
7838}
7839
7840Register fastEmit_AArch64ISD_SQDMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7841 switch (VT.SimpleTy) {
7842 case MVT::f32: return fastEmit_AArch64ISD_SQDMULL_MVT_f32_rr(RetVT, Op0, Op1);
7843 default: return Register();
7844 }
7845}
7846
7847// FastEmit functions for AArch64ISD::SQRDMULH.
7848
7849Register fastEmit_AArch64ISD_SQRDMULH_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7850 if (RetVT.SimpleTy != MVT::f32)
7851 return Register();
7852 if ((Subtarget->isNeonAvailable())) {
7853 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRDMULHv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7854 }
7855 return Register();
7856}
7857
7858Register fastEmit_AArch64ISD_SQRDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7859 switch (VT.SimpleTy) {
7860 case MVT::f32: return fastEmit_AArch64ISD_SQRDMULH_MVT_f32_rr(RetVT, Op0, Op1);
7861 default: return Register();
7862 }
7863}
7864
7865// FastEmit functions for AArch64ISD::SQRSHL.
7866
7867Register fastEmit_AArch64ISD_SQRSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7868 if (RetVT.SimpleTy != MVT::f32)
7869 return Register();
7870 if ((Subtarget->isNeonAvailable())) {
7871 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7872 }
7873 return Register();
7874}
7875
7876Register fastEmit_AArch64ISD_SQRSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7877 if (RetVT.SimpleTy != MVT::f64)
7878 return Register();
7879 if ((Subtarget->isNeonAvailable())) {
7880 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7881 }
7882 return Register();
7883}
7884
7885Register fastEmit_AArch64ISD_SQRSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7886 switch (VT.SimpleTy) {
7887 case MVT::f32: return fastEmit_AArch64ISD_SQRSHL_MVT_f32_rr(RetVT, Op0, Op1);
7888 case MVT::f64: return fastEmit_AArch64ISD_SQRSHL_MVT_f64_rr(RetVT, Op0, Op1);
7889 default: return Register();
7890 }
7891}
7892
7893// FastEmit functions for AArch64ISD::SQSHL.
7894
7895Register fastEmit_AArch64ISD_SQSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7896 if (RetVT.SimpleTy != MVT::f32)
7897 return Register();
7898 if ((Subtarget->isNeonAvailable())) {
7899 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7900 }
7901 return Register();
7902}
7903
7904Register fastEmit_AArch64ISD_SQSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7905 if (RetVT.SimpleTy != MVT::f64)
7906 return Register();
7907 if ((Subtarget->isNeonAvailable())) {
7908 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7909 }
7910 return Register();
7911}
7912
7913Register fastEmit_AArch64ISD_SQSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7914 switch (VT.SimpleTy) {
7915 case MVT::f32: return fastEmit_AArch64ISD_SQSHL_MVT_f32_rr(RetVT, Op0, Op1);
7916 case MVT::f64: return fastEmit_AArch64ISD_SQSHL_MVT_f64_rr(RetVT, Op0, Op1);
7917 default: return Register();
7918 }
7919}
7920
7921// FastEmit functions for AArch64ISD::SQSUB.
7922
7923Register fastEmit_AArch64ISD_SQSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7924 if (RetVT.SimpleTy != MVT::f32)
7925 return Register();
7926 if ((Subtarget->isNeonAvailable())) {
7927 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7928 }
7929 return Register();
7930}
7931
7932Register fastEmit_AArch64ISD_SQSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7933 if (RetVT.SimpleTy != MVT::f64)
7934 return Register();
7935 if ((Subtarget->isNeonAvailable())) {
7936 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7937 }
7938 return Register();
7939}
7940
7941Register fastEmit_AArch64ISD_SQSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7942 switch (VT.SimpleTy) {
7943 case MVT::f32: return fastEmit_AArch64ISD_SQSUB_MVT_f32_rr(RetVT, Op0, Op1);
7944 case MVT::f64: return fastEmit_AArch64ISD_SQSUB_MVT_f64_rr(RetVT, Op0, Op1);
7945 default: return Register();
7946 }
7947}
7948
7949// FastEmit functions for AArch64ISD::STRICT_FCMP.
7950
7951Register fastEmit_AArch64ISD_STRICT_FCMP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
7952 if (RetVT.SimpleTy != MVT::i32)
7953 return Register();
7954 if ((Subtarget->hasFullFP16())) {
7955 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
7956 }
7957 return Register();
7958}
7959
7960Register fastEmit_AArch64ISD_STRICT_FCMP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7961 if (RetVT.SimpleTy != MVT::i32)
7962 return Register();
7963 if ((Subtarget->hasFPARMv8())) {
7964 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
7965 }
7966 return Register();
7967}
7968
7969Register fastEmit_AArch64ISD_STRICT_FCMP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7970 if (RetVT.SimpleTy != MVT::i32)
7971 return Register();
7972 if ((Subtarget->hasFPARMv8())) {
7973 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
7974 }
7975 return Register();
7976}
7977
7978Register fastEmit_AArch64ISD_STRICT_FCMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7979 switch (VT.SimpleTy) {
7980 case MVT::f16: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
7981 case MVT::f32: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
7982 case MVT::f64: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
7983 default: return Register();
7984 }
7985}
7986
7987// FastEmit functions for AArch64ISD::STRICT_FCMPE.
7988
7989Register fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
7990 if (RetVT.SimpleTy != MVT::i32)
7991 return Register();
7992 if ((Subtarget->hasFullFP16())) {
7993 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPEHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
7994 }
7995 return Register();
7996}
7997
7998Register fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7999 if (RetVT.SimpleTy != MVT::i32)
8000 return Register();
8001 if ((Subtarget->hasFPARMv8())) {
8002 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPESrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
8003 }
8004 return Register();
8005}
8006
8007Register fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8008 if (RetVT.SimpleTy != MVT::i32)
8009 return Register();
8010 if ((Subtarget->hasFPARMv8())) {
8011 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPEDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
8012 }
8013 return Register();
8014}
8015
8016Register fastEmit_AArch64ISD_STRICT_FCMPE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8017 switch (VT.SimpleTy) {
8018 case MVT::f16: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f16_rr(RetVT, Op0, Op1);
8019 case MVT::f32: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f32_rr(RetVT, Op0, Op1);
8020 case MVT::f64: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f64_rr(RetVT, Op0, Op1);
8021 default: return Register();
8022 }
8023}
8024
8025// FastEmit functions for AArch64ISD::SUQADD.
8026
8027Register fastEmit_AArch64ISD_SUQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8028 if (RetVT.SimpleTy != MVT::f32)
8029 return Register();
8030 if ((Subtarget->isNeonAvailable())) {
8031 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8032 }
8033 return Register();
8034}
8035
8036Register fastEmit_AArch64ISD_SUQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8037 if (RetVT.SimpleTy != MVT::f64)
8038 return Register();
8039 if ((Subtarget->isNeonAvailable())) {
8040 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8041 }
8042 return Register();
8043}
8044
8045Register fastEmit_AArch64ISD_SUQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8046 switch (VT.SimpleTy) {
8047 case MVT::f32: return fastEmit_AArch64ISD_SUQADD_MVT_f32_rr(RetVT, Op0, Op1);
8048 case MVT::f64: return fastEmit_AArch64ISD_SUQADD_MVT_f64_rr(RetVT, Op0, Op1);
8049 default: return Register();
8050 }
8051}
8052
8053// FastEmit functions for AArch64ISD::TBL.
8054
8055Register fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8056 if (RetVT.SimpleTy != MVT::nxv16i8)
8057 return Register();
8058 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8059 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8060 }
8061 return Register();
8062}
8063
8064Register fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8065 if (RetVT.SimpleTy != MVT::nxv8i16)
8066 return Register();
8067 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8068 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8069 }
8070 return Register();
8071}
8072
8073Register fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8074 if (RetVT.SimpleTy != MVT::nxv4i32)
8075 return Register();
8076 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8077 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8078 }
8079 return Register();
8080}
8081
8082Register fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8083 if (RetVT.SimpleTy != MVT::nxv2i64)
8084 return Register();
8085 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8086 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8087 }
8088 return Register();
8089}
8090
8091Register fastEmit_AArch64ISD_TBL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8092 switch (VT.SimpleTy) {
8093 case MVT::nxv16i8: return fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8094 case MVT::nxv8i16: return fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8095 case MVT::nxv4i32: return fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8096 case MVT::nxv2i64: return fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8097 default: return Register();
8098 }
8099}
8100
8101// FastEmit functions for AArch64ISD::TRN1.
8102
8103Register fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8104 if (RetVT.SimpleTy != MVT::v8i8)
8105 return Register();
8106 if ((Subtarget->isNeonAvailable())) {
8107 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8108 }
8109 return Register();
8110}
8111
8112Register fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8113 if (RetVT.SimpleTy != MVT::v16i8)
8114 return Register();
8115 if ((Subtarget->isNeonAvailable())) {
8116 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8117 }
8118 return Register();
8119}
8120
8121Register fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8122 if (RetVT.SimpleTy != MVT::v4i16)
8123 return Register();
8124 if ((Subtarget->isNeonAvailable())) {
8125 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8126 }
8127 return Register();
8128}
8129
8130Register fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8131 if (RetVT.SimpleTy != MVT::v8i16)
8132 return Register();
8133 if ((Subtarget->isNeonAvailable())) {
8134 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8135 }
8136 return Register();
8137}
8138
8139Register fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8140 if (RetVT.SimpleTy != MVT::v2i32)
8141 return Register();
8142 if ((Subtarget->isNeonAvailable())) {
8143 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8144 }
8145 return Register();
8146}
8147
8148Register fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8149 if (RetVT.SimpleTy != MVT::v4i32)
8150 return Register();
8151 if ((Subtarget->isNeonAvailable())) {
8152 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8153 }
8154 return Register();
8155}
8156
8157Register fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8158 if (RetVT.SimpleTy != MVT::v2i64)
8159 return Register();
8160 if ((Subtarget->isNeonAvailable())) {
8161 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8162 }
8163 return Register();
8164}
8165
8166Register fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8167 if (RetVT.SimpleTy != MVT::v4f16)
8168 return Register();
8169 if ((Subtarget->isNeonAvailable())) {
8170 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8171 }
8172 return Register();
8173}
8174
8175Register fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8176 if (RetVT.SimpleTy != MVT::v8f16)
8177 return Register();
8178 if ((Subtarget->isNeonAvailable())) {
8179 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8180 }
8181 return Register();
8182}
8183
8184Register fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8185 if (RetVT.SimpleTy != MVT::v4bf16)
8186 return Register();
8187 if ((Subtarget->isNeonAvailable())) {
8188 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8189 }
8190 return Register();
8191}
8192
8193Register fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8194 if (RetVT.SimpleTy != MVT::v8bf16)
8195 return Register();
8196 if ((Subtarget->isNeonAvailable())) {
8197 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8198 }
8199 return Register();
8200}
8201
8202Register fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8203 if (RetVT.SimpleTy != MVT::v2f32)
8204 return Register();
8205 if ((Subtarget->isNeonAvailable())) {
8206 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8207 }
8208 return Register();
8209}
8210
8211Register fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8212 if (RetVT.SimpleTy != MVT::v4f32)
8213 return Register();
8214 if ((Subtarget->isNeonAvailable())) {
8215 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8216 }
8217 return Register();
8218}
8219
8220Register fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8221 if (RetVT.SimpleTy != MVT::v2f64)
8222 return Register();
8223 if ((Subtarget->isNeonAvailable())) {
8224 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8225 }
8226 return Register();
8227}
8228
8229Register fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8230 if (RetVT.SimpleTy != MVT::nxv2i1)
8231 return Register();
8232 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8233 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8234 }
8235 return Register();
8236}
8237
8238Register fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8239 if (RetVT.SimpleTy != MVT::nxv4i1)
8240 return Register();
8241 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8242 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8243 }
8244 return Register();
8245}
8246
8247Register fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8248 if (RetVT.SimpleTy != MVT::nxv8i1)
8249 return Register();
8250 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8251 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8252 }
8253 return Register();
8254}
8255
8256Register fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8257 if (RetVT.SimpleTy != MVT::nxv16i1)
8258 return Register();
8259 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8260 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8261 }
8262 return Register();
8263}
8264
8265Register fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8266 if (RetVT.SimpleTy != MVT::nxv16i8)
8267 return Register();
8268 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8269 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8270 }
8271 return Register();
8272}
8273
8274Register fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8275 if (RetVT.SimpleTy != MVT::nxv8i16)
8276 return Register();
8277 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8278 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8279 }
8280 return Register();
8281}
8282
8283Register fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8284 if (RetVT.SimpleTy != MVT::nxv4i32)
8285 return Register();
8286 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8287 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8288 }
8289 return Register();
8290}
8291
8292Register fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8293 if (RetVT.SimpleTy != MVT::nxv2i64)
8294 return Register();
8295 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8296 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8297 }
8298 return Register();
8299}
8300
8301Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8302 if (RetVT.SimpleTy != MVT::nxv2f16)
8303 return Register();
8304 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8305 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8306 }
8307 return Register();
8308}
8309
8310Register fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8311 if (RetVT.SimpleTy != MVT::nxv4f16)
8312 return Register();
8313 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8314 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8315 }
8316 return Register();
8317}
8318
8319Register fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8320 if (RetVT.SimpleTy != MVT::nxv8f16)
8321 return Register();
8322 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8323 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8324 }
8325 return Register();
8326}
8327
8328Register fastEmit_AArch64ISD_TRN1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8329 if (RetVT.SimpleTy != MVT::nxv2bf16)
8330 return Register();
8331 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8332 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8333 }
8334 return Register();
8335}
8336
8337Register fastEmit_AArch64ISD_TRN1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8338 if (RetVT.SimpleTy != MVT::nxv4bf16)
8339 return Register();
8340 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8341 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8342 }
8343 return Register();
8344}
8345
8346Register fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8347 if (RetVT.SimpleTy != MVT::nxv8bf16)
8348 return Register();
8349 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8350 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8351 }
8352 return Register();
8353}
8354
8355Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8356 if (RetVT.SimpleTy != MVT::nxv2f32)
8357 return Register();
8358 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8359 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8360 }
8361 return Register();
8362}
8363
8364Register fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8365 if (RetVT.SimpleTy != MVT::nxv4f32)
8366 return Register();
8367 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8368 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8369 }
8370 return Register();
8371}
8372
8373Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8374 if (RetVT.SimpleTy != MVT::nxv2f64)
8375 return Register();
8376 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8377 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8378 }
8379 return Register();
8380}
8381
8382Register fastEmit_AArch64ISD_TRN1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8383 switch (VT.SimpleTy) {
8384 case MVT::v8i8: return fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(RetVT, Op0, Op1);
8385 case MVT::v16i8: return fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(RetVT, Op0, Op1);
8386 case MVT::v4i16: return fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(RetVT, Op0, Op1);
8387 case MVT::v8i16: return fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(RetVT, Op0, Op1);
8388 case MVT::v2i32: return fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(RetVT, Op0, Op1);
8389 case MVT::v4i32: return fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(RetVT, Op0, Op1);
8390 case MVT::v2i64: return fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(RetVT, Op0, Op1);
8391 case MVT::v4f16: return fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(RetVT, Op0, Op1);
8392 case MVT::v8f16: return fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(RetVT, Op0, Op1);
8393 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(RetVT, Op0, Op1);
8394 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(RetVT, Op0, Op1);
8395 case MVT::v2f32: return fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(RetVT, Op0, Op1);
8396 case MVT::v4f32: return fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(RetVT, Op0, Op1);
8397 case MVT::v2f64: return fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(RetVT, Op0, Op1);
8398 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8399 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8400 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8401 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8402 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8403 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8404 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8405 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8406 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8407 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8408 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8409 case MVT::nxv2bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8410 case MVT::nxv4bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8411 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8412 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8413 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8414 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8415 default: return Register();
8416 }
8417}
8418
8419// FastEmit functions for AArch64ISD::TRN2.
8420
8421Register fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8422 if (RetVT.SimpleTy != MVT::v8i8)
8423 return Register();
8424 if ((Subtarget->isNeonAvailable())) {
8425 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8426 }
8427 return Register();
8428}
8429
8430Register fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8431 if (RetVT.SimpleTy != MVT::v16i8)
8432 return Register();
8433 if ((Subtarget->isNeonAvailable())) {
8434 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8435 }
8436 return Register();
8437}
8438
8439Register fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8440 if (RetVT.SimpleTy != MVT::v4i16)
8441 return Register();
8442 if ((Subtarget->isNeonAvailable())) {
8443 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8444 }
8445 return Register();
8446}
8447
8448Register fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8449 if (RetVT.SimpleTy != MVT::v8i16)
8450 return Register();
8451 if ((Subtarget->isNeonAvailable())) {
8452 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8453 }
8454 return Register();
8455}
8456
8457Register fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8458 if (RetVT.SimpleTy != MVT::v2i32)
8459 return Register();
8460 if ((Subtarget->isNeonAvailable())) {
8461 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8462 }
8463 return Register();
8464}
8465
8466Register fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8467 if (RetVT.SimpleTy != MVT::v4i32)
8468 return Register();
8469 if ((Subtarget->isNeonAvailable())) {
8470 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8471 }
8472 return Register();
8473}
8474
8475Register fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8476 if (RetVT.SimpleTy != MVT::v2i64)
8477 return Register();
8478 if ((Subtarget->isNeonAvailable())) {
8479 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8480 }
8481 return Register();
8482}
8483
8484Register fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8485 if (RetVT.SimpleTy != MVT::v4f16)
8486 return Register();
8487 if ((Subtarget->isNeonAvailable())) {
8488 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8489 }
8490 return Register();
8491}
8492
8493Register fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8494 if (RetVT.SimpleTy != MVT::v8f16)
8495 return Register();
8496 if ((Subtarget->isNeonAvailable())) {
8497 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8498 }
8499 return Register();
8500}
8501
8502Register fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8503 if (RetVT.SimpleTy != MVT::v4bf16)
8504 return Register();
8505 if ((Subtarget->isNeonAvailable())) {
8506 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8507 }
8508 return Register();
8509}
8510
8511Register fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8512 if (RetVT.SimpleTy != MVT::v8bf16)
8513 return Register();
8514 if ((Subtarget->isNeonAvailable())) {
8515 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8516 }
8517 return Register();
8518}
8519
8520Register fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8521 if (RetVT.SimpleTy != MVT::v2f32)
8522 return Register();
8523 if ((Subtarget->isNeonAvailable())) {
8524 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8525 }
8526 return Register();
8527}
8528
8529Register fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8530 if (RetVT.SimpleTy != MVT::v4f32)
8531 return Register();
8532 if ((Subtarget->isNeonAvailable())) {
8533 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8534 }
8535 return Register();
8536}
8537
8538Register fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8539 if (RetVT.SimpleTy != MVT::v2f64)
8540 return Register();
8541 if ((Subtarget->isNeonAvailable())) {
8542 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8543 }
8544 return Register();
8545}
8546
8547Register fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8548 if (RetVT.SimpleTy != MVT::nxv2i1)
8549 return Register();
8550 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8551 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8552 }
8553 return Register();
8554}
8555
8556Register fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8557 if (RetVT.SimpleTy != MVT::nxv4i1)
8558 return Register();
8559 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8560 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8561 }
8562 return Register();
8563}
8564
8565Register fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8566 if (RetVT.SimpleTy != MVT::nxv8i1)
8567 return Register();
8568 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8569 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8570 }
8571 return Register();
8572}
8573
8574Register fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8575 if (RetVT.SimpleTy != MVT::nxv16i1)
8576 return Register();
8577 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8578 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8579 }
8580 return Register();
8581}
8582
8583Register fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8584 if (RetVT.SimpleTy != MVT::nxv16i8)
8585 return Register();
8586 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8587 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8588 }
8589 return Register();
8590}
8591
8592Register fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8593 if (RetVT.SimpleTy != MVT::nxv8i16)
8594 return Register();
8595 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8596 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8597 }
8598 return Register();
8599}
8600
8601Register fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8602 if (RetVT.SimpleTy != MVT::nxv4i32)
8603 return Register();
8604 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8605 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8606 }
8607 return Register();
8608}
8609
8610Register fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8611 if (RetVT.SimpleTy != MVT::nxv2i64)
8612 return Register();
8613 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8614 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8615 }
8616 return Register();
8617}
8618
8619Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8620 if (RetVT.SimpleTy != MVT::nxv2f16)
8621 return Register();
8622 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8623 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8624 }
8625 return Register();
8626}
8627
8628Register fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8629 if (RetVT.SimpleTy != MVT::nxv4f16)
8630 return Register();
8631 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8632 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8633 }
8634 return Register();
8635}
8636
8637Register fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8638 if (RetVT.SimpleTy != MVT::nxv8f16)
8639 return Register();
8640 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8641 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8642 }
8643 return Register();
8644}
8645
8646Register fastEmit_AArch64ISD_TRN2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8647 if (RetVT.SimpleTy != MVT::nxv2bf16)
8648 return Register();
8649 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8650 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8651 }
8652 return Register();
8653}
8654
8655Register fastEmit_AArch64ISD_TRN2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8656 if (RetVT.SimpleTy != MVT::nxv4bf16)
8657 return Register();
8658 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8659 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8660 }
8661 return Register();
8662}
8663
8664Register fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8665 if (RetVT.SimpleTy != MVT::nxv8bf16)
8666 return Register();
8667 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8668 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8669 }
8670 return Register();
8671}
8672
8673Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8674 if (RetVT.SimpleTy != MVT::nxv2f32)
8675 return Register();
8676 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8677 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8678 }
8679 return Register();
8680}
8681
8682Register fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8683 if (RetVT.SimpleTy != MVT::nxv4f32)
8684 return Register();
8685 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8686 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8687 }
8688 return Register();
8689}
8690
8691Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8692 if (RetVT.SimpleTy != MVT::nxv2f64)
8693 return Register();
8694 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8695 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8696 }
8697 return Register();
8698}
8699
8700Register fastEmit_AArch64ISD_TRN2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8701 switch (VT.SimpleTy) {
8702 case MVT::v8i8: return fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(RetVT, Op0, Op1);
8703 case MVT::v16i8: return fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(RetVT, Op0, Op1);
8704 case MVT::v4i16: return fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(RetVT, Op0, Op1);
8705 case MVT::v8i16: return fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(RetVT, Op0, Op1);
8706 case MVT::v2i32: return fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(RetVT, Op0, Op1);
8707 case MVT::v4i32: return fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(RetVT, Op0, Op1);
8708 case MVT::v2i64: return fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(RetVT, Op0, Op1);
8709 case MVT::v4f16: return fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(RetVT, Op0, Op1);
8710 case MVT::v8f16: return fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(RetVT, Op0, Op1);
8711 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(RetVT, Op0, Op1);
8712 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(RetVT, Op0, Op1);
8713 case MVT::v2f32: return fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(RetVT, Op0, Op1);
8714 case MVT::v4f32: return fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(RetVT, Op0, Op1);
8715 case MVT::v2f64: return fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(RetVT, Op0, Op1);
8716 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8717 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8718 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8719 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8720 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8721 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8722 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8723 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8724 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8725 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8726 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8727 case MVT::nxv2bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8728 case MVT::nxv4bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8729 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8730 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8731 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8732 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8733 default: return Register();
8734 }
8735}
8736
8737// FastEmit functions for AArch64ISD::UMULL.
8738
8739Register fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8740 if (RetVT.SimpleTy != MVT::v8i16)
8741 return Register();
8742 if ((Subtarget->isNeonAvailable())) {
8743 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv8i8_v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8744 }
8745 return Register();
8746}
8747
8748Register fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8749 if (RetVT.SimpleTy != MVT::v4i32)
8750 return Register();
8751 if ((Subtarget->isNeonAvailable())) {
8752 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv4i16_v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8753 }
8754 return Register();
8755}
8756
8757Register fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8758 if (RetVT.SimpleTy != MVT::v2i64)
8759 return Register();
8760 if ((Subtarget->isNeonAvailable())) {
8761 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv2i32_v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8762 }
8763 return Register();
8764}
8765
8766Register fastEmit_AArch64ISD_UMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8767 switch (VT.SimpleTy) {
8768 case MVT::v8i8: return fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
8769 case MVT::v4i16: return fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(RetVT, Op0, Op1);
8770 case MVT::v2i32: return fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(RetVT, Op0, Op1);
8771 default: return Register();
8772 }
8773}
8774
8775// FastEmit functions for AArch64ISD::UQADD.
8776
8777Register fastEmit_AArch64ISD_UQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8778 if (RetVT.SimpleTy != MVT::f32)
8779 return Register();
8780 if ((Subtarget->isNeonAvailable())) {
8781 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8782 }
8783 return Register();
8784}
8785
8786Register fastEmit_AArch64ISD_UQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8787 if (RetVT.SimpleTy != MVT::f64)
8788 return Register();
8789 if ((Subtarget->isNeonAvailable())) {
8790 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8791 }
8792 return Register();
8793}
8794
8795Register fastEmit_AArch64ISD_UQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8796 switch (VT.SimpleTy) {
8797 case MVT::f32: return fastEmit_AArch64ISD_UQADD_MVT_f32_rr(RetVT, Op0, Op1);
8798 case MVT::f64: return fastEmit_AArch64ISD_UQADD_MVT_f64_rr(RetVT, Op0, Op1);
8799 default: return Register();
8800 }
8801}
8802
8803// FastEmit functions for AArch64ISD::UQRSHL.
8804
8805Register fastEmit_AArch64ISD_UQRSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8806 if (RetVT.SimpleTy != MVT::f32)
8807 return Register();
8808 if ((Subtarget->isNeonAvailable())) {
8809 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQRSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8810 }
8811 return Register();
8812}
8813
8814Register fastEmit_AArch64ISD_UQRSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8815 if (RetVT.SimpleTy != MVT::f64)
8816 return Register();
8817 if ((Subtarget->isNeonAvailable())) {
8818 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQRSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8819 }
8820 return Register();
8821}
8822
8823Register fastEmit_AArch64ISD_UQRSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8824 switch (VT.SimpleTy) {
8825 case MVT::f32: return fastEmit_AArch64ISD_UQRSHL_MVT_f32_rr(RetVT, Op0, Op1);
8826 case MVT::f64: return fastEmit_AArch64ISD_UQRSHL_MVT_f64_rr(RetVT, Op0, Op1);
8827 default: return Register();
8828 }
8829}
8830
8831// FastEmit functions for AArch64ISD::UQSHL.
8832
8833Register fastEmit_AArch64ISD_UQSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8834 if (RetVT.SimpleTy != MVT::f32)
8835 return Register();
8836 if ((Subtarget->isNeonAvailable())) {
8837 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8838 }
8839 return Register();
8840}
8841
8842Register fastEmit_AArch64ISD_UQSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8843 if (RetVT.SimpleTy != MVT::f64)
8844 return Register();
8845 if ((Subtarget->isNeonAvailable())) {
8846 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8847 }
8848 return Register();
8849}
8850
8851Register fastEmit_AArch64ISD_UQSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8852 switch (VT.SimpleTy) {
8853 case MVT::f32: return fastEmit_AArch64ISD_UQSHL_MVT_f32_rr(RetVT, Op0, Op1);
8854 case MVT::f64: return fastEmit_AArch64ISD_UQSHL_MVT_f64_rr(RetVT, Op0, Op1);
8855 default: return Register();
8856 }
8857}
8858
8859// FastEmit functions for AArch64ISD::UQSUB.
8860
8861Register fastEmit_AArch64ISD_UQSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8862 if (RetVT.SimpleTy != MVT::f32)
8863 return Register();
8864 if ((Subtarget->isNeonAvailable())) {
8865 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8866 }
8867 return Register();
8868}
8869
8870Register fastEmit_AArch64ISD_UQSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8871 if (RetVT.SimpleTy != MVT::f64)
8872 return Register();
8873 if ((Subtarget->isNeonAvailable())) {
8874 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8875 }
8876 return Register();
8877}
8878
8879Register fastEmit_AArch64ISD_UQSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8880 switch (VT.SimpleTy) {
8881 case MVT::f32: return fastEmit_AArch64ISD_UQSUB_MVT_f32_rr(RetVT, Op0, Op1);
8882 case MVT::f64: return fastEmit_AArch64ISD_UQSUB_MVT_f64_rr(RetVT, Op0, Op1);
8883 default: return Register();
8884 }
8885}
8886
8887// FastEmit functions for AArch64ISD::USQADD.
8888
8889Register fastEmit_AArch64ISD_USQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8890 if (RetVT.SimpleTy != MVT::f32)
8891 return Register();
8892 if ((Subtarget->isNeonAvailable())) {
8893 return fastEmitInst_rr(MachineInstOpcode: AArch64::USQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8894 }
8895 return Register();
8896}
8897
8898Register fastEmit_AArch64ISD_USQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8899 if (RetVT.SimpleTy != MVT::f64)
8900 return Register();
8901 if ((Subtarget->isNeonAvailable())) {
8902 return fastEmitInst_rr(MachineInstOpcode: AArch64::USQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8903 }
8904 return Register();
8905}
8906
8907Register fastEmit_AArch64ISD_USQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8908 switch (VT.SimpleTy) {
8909 case MVT::f32: return fastEmit_AArch64ISD_USQADD_MVT_f32_rr(RetVT, Op0, Op1);
8910 case MVT::f64: return fastEmit_AArch64ISD_USQADD_MVT_f64_rr(RetVT, Op0, Op1);
8911 default: return Register();
8912 }
8913}
8914
8915// FastEmit functions for AArch64ISD::UZP1.
8916
8917Register fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8918 if (RetVT.SimpleTy != MVT::v8i8)
8919 return Register();
8920 if ((Subtarget->isNeonAvailable())) {
8921 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8922 }
8923 return Register();
8924}
8925
8926Register fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8927 if (RetVT.SimpleTy != MVT::v16i8)
8928 return Register();
8929 if ((Subtarget->isNeonAvailable())) {
8930 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8931 }
8932 return Register();
8933}
8934
8935Register fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8936 if (RetVT.SimpleTy != MVT::v4i16)
8937 return Register();
8938 if ((Subtarget->isNeonAvailable())) {
8939 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8940 }
8941 return Register();
8942}
8943
8944Register fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8945 if (RetVT.SimpleTy != MVT::v8i16)
8946 return Register();
8947 if ((Subtarget->isNeonAvailable())) {
8948 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8949 }
8950 return Register();
8951}
8952
8953Register fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8954 if (RetVT.SimpleTy != MVT::v2i32)
8955 return Register();
8956 if ((Subtarget->isNeonAvailable())) {
8957 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8958 }
8959 return Register();
8960}
8961
8962Register fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8963 if (RetVT.SimpleTy != MVT::v4i32)
8964 return Register();
8965 if ((Subtarget->isNeonAvailable())) {
8966 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8967 }
8968 return Register();
8969}
8970
8971Register fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8972 if (RetVT.SimpleTy != MVT::v2i64)
8973 return Register();
8974 if ((Subtarget->isNeonAvailable())) {
8975 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8976 }
8977 return Register();
8978}
8979
8980Register fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8981 if (RetVT.SimpleTy != MVT::v4f16)
8982 return Register();
8983 if ((Subtarget->isNeonAvailable())) {
8984 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8985 }
8986 return Register();
8987}
8988
8989Register fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8990 if (RetVT.SimpleTy != MVT::v8f16)
8991 return Register();
8992 if ((Subtarget->isNeonAvailable())) {
8993 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8994 }
8995 return Register();
8996}
8997
8998Register fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8999 if (RetVT.SimpleTy != MVT::v4bf16)
9000 return Register();
9001 if ((Subtarget->isNeonAvailable())) {
9002 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9003 }
9004 return Register();
9005}
9006
9007Register fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9008 if (RetVT.SimpleTy != MVT::v8bf16)
9009 return Register();
9010 if ((Subtarget->isNeonAvailable())) {
9011 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9012 }
9013 return Register();
9014}
9015
9016Register fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9017 if (RetVT.SimpleTy != MVT::v2f32)
9018 return Register();
9019 if ((Subtarget->isNeonAvailable())) {
9020 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9021 }
9022 return Register();
9023}
9024
9025Register fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9026 if (RetVT.SimpleTy != MVT::v4f32)
9027 return Register();
9028 if ((Subtarget->isNeonAvailable())) {
9029 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9030 }
9031 return Register();
9032}
9033
9034Register fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9035 if (RetVT.SimpleTy != MVT::v2f64)
9036 return Register();
9037 if ((Subtarget->isNeonAvailable())) {
9038 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9039 }
9040 return Register();
9041}
9042
9043Register fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9044 if (RetVT.SimpleTy != MVT::nxv2i1)
9045 return Register();
9046 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9047 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9048 }
9049 return Register();
9050}
9051
9052Register fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9053 if (RetVT.SimpleTy != MVT::nxv4i1)
9054 return Register();
9055 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9056 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9057 }
9058 return Register();
9059}
9060
9061Register fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9062 if (RetVT.SimpleTy != MVT::nxv8i1)
9063 return Register();
9064 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9065 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9066 }
9067 return Register();
9068}
9069
9070Register fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9071 if (RetVT.SimpleTy != MVT::nxv16i1)
9072 return Register();
9073 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9074 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9075 }
9076 return Register();
9077}
9078
9079Register fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9080 if (RetVT.SimpleTy != MVT::nxv16i8)
9081 return Register();
9082 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9083 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9084 }
9085 return Register();
9086}
9087
9088Register fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9089 if (RetVT.SimpleTy != MVT::nxv8i16)
9090 return Register();
9091 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9092 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9093 }
9094 return Register();
9095}
9096
9097Register fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9098 if (RetVT.SimpleTy != MVT::nxv4i32)
9099 return Register();
9100 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9101 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9102 }
9103 return Register();
9104}
9105
9106Register fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9107 if (RetVT.SimpleTy != MVT::nxv2i64)
9108 return Register();
9109 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9110 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9111 }
9112 return Register();
9113}
9114
9115Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9116 if (RetVT.SimpleTy != MVT::nxv2f16)
9117 return Register();
9118 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9119 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9120 }
9121 return Register();
9122}
9123
9124Register fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9125 if (RetVT.SimpleTy != MVT::nxv4f16)
9126 return Register();
9127 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9128 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9129 }
9130 return Register();
9131}
9132
9133Register fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9134 if (RetVT.SimpleTy != MVT::nxv8f16)
9135 return Register();
9136 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9137 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9138 }
9139 return Register();
9140}
9141
9142Register fastEmit_AArch64ISD_UZP1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9143 if (RetVT.SimpleTy != MVT::nxv2bf16)
9144 return Register();
9145 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9146 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9147 }
9148 return Register();
9149}
9150
9151Register fastEmit_AArch64ISD_UZP1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9152 if (RetVT.SimpleTy != MVT::nxv4bf16)
9153 return Register();
9154 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9155 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9156 }
9157 return Register();
9158}
9159
9160Register fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9161 if (RetVT.SimpleTy != MVT::nxv8bf16)
9162 return Register();
9163 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9164 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9165 }
9166 return Register();
9167}
9168
9169Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9170 if (RetVT.SimpleTy != MVT::nxv2f32)
9171 return Register();
9172 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9173 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9174 }
9175 return Register();
9176}
9177
9178Register fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9179 if (RetVT.SimpleTy != MVT::nxv4f32)
9180 return Register();
9181 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9182 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9183 }
9184 return Register();
9185}
9186
9187Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9188 if (RetVT.SimpleTy != MVT::nxv2f64)
9189 return Register();
9190 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9191 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9192 }
9193 return Register();
9194}
9195
9196Register fastEmit_AArch64ISD_UZP1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9197 switch (VT.SimpleTy) {
9198 case MVT::v8i8: return fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(RetVT, Op0, Op1);
9199 case MVT::v16i8: return fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(RetVT, Op0, Op1);
9200 case MVT::v4i16: return fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(RetVT, Op0, Op1);
9201 case MVT::v8i16: return fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(RetVT, Op0, Op1);
9202 case MVT::v2i32: return fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(RetVT, Op0, Op1);
9203 case MVT::v4i32: return fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(RetVT, Op0, Op1);
9204 case MVT::v2i64: return fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(RetVT, Op0, Op1);
9205 case MVT::v4f16: return fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(RetVT, Op0, Op1);
9206 case MVT::v8f16: return fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(RetVT, Op0, Op1);
9207 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(RetVT, Op0, Op1);
9208 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(RetVT, Op0, Op1);
9209 case MVT::v2f32: return fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(RetVT, Op0, Op1);
9210 case MVT::v4f32: return fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(RetVT, Op0, Op1);
9211 case MVT::v2f64: return fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(RetVT, Op0, Op1);
9212 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9213 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9214 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9215 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9216 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9217 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9218 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9219 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9220 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9221 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9222 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9223 case MVT::nxv2bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9224 case MVT::nxv4bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9225 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9226 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9227 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9228 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9229 default: return Register();
9230 }
9231}
9232
9233// FastEmit functions for AArch64ISD::UZP2.
9234
9235Register fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9236 if (RetVT.SimpleTy != MVT::v8i8)
9237 return Register();
9238 if ((Subtarget->isNeonAvailable())) {
9239 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9240 }
9241 return Register();
9242}
9243
9244Register fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9245 if (RetVT.SimpleTy != MVT::v16i8)
9246 return Register();
9247 if ((Subtarget->isNeonAvailable())) {
9248 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9249 }
9250 return Register();
9251}
9252
9253Register fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9254 if (RetVT.SimpleTy != MVT::v4i16)
9255 return Register();
9256 if ((Subtarget->isNeonAvailable())) {
9257 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9258 }
9259 return Register();
9260}
9261
9262Register fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9263 if (RetVT.SimpleTy != MVT::v8i16)
9264 return Register();
9265 if ((Subtarget->isNeonAvailable())) {
9266 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9267 }
9268 return Register();
9269}
9270
9271Register fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9272 if (RetVT.SimpleTy != MVT::v2i32)
9273 return Register();
9274 if ((Subtarget->isNeonAvailable())) {
9275 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9276 }
9277 return Register();
9278}
9279
9280Register fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9281 if (RetVT.SimpleTy != MVT::v4i32)
9282 return Register();
9283 if ((Subtarget->isNeonAvailable())) {
9284 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9285 }
9286 return Register();
9287}
9288
9289Register fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9290 if (RetVT.SimpleTy != MVT::v2i64)
9291 return Register();
9292 if ((Subtarget->isNeonAvailable())) {
9293 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9294 }
9295 return Register();
9296}
9297
9298Register fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9299 if (RetVT.SimpleTy != MVT::v4f16)
9300 return Register();
9301 if ((Subtarget->isNeonAvailable())) {
9302 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9303 }
9304 return Register();
9305}
9306
9307Register fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9308 if (RetVT.SimpleTy != MVT::v8f16)
9309 return Register();
9310 if ((Subtarget->isNeonAvailable())) {
9311 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9312 }
9313 return Register();
9314}
9315
9316Register fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9317 if (RetVT.SimpleTy != MVT::v4bf16)
9318 return Register();
9319 if ((Subtarget->isNeonAvailable())) {
9320 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9321 }
9322 return Register();
9323}
9324
9325Register fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9326 if (RetVT.SimpleTy != MVT::v8bf16)
9327 return Register();
9328 if ((Subtarget->isNeonAvailable())) {
9329 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9330 }
9331 return Register();
9332}
9333
9334Register fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9335 if (RetVT.SimpleTy != MVT::v2f32)
9336 return Register();
9337 if ((Subtarget->isNeonAvailable())) {
9338 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9339 }
9340 return Register();
9341}
9342
9343Register fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9344 if (RetVT.SimpleTy != MVT::v4f32)
9345 return Register();
9346 if ((Subtarget->isNeonAvailable())) {
9347 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9348 }
9349 return Register();
9350}
9351
9352Register fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9353 if (RetVT.SimpleTy != MVT::v2f64)
9354 return Register();
9355 if ((Subtarget->isNeonAvailable())) {
9356 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9357 }
9358 return Register();
9359}
9360
9361Register fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9362 if (RetVT.SimpleTy != MVT::nxv2i1)
9363 return Register();
9364 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9365 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9366 }
9367 return Register();
9368}
9369
9370Register fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9371 if (RetVT.SimpleTy != MVT::nxv4i1)
9372 return Register();
9373 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9374 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9375 }
9376 return Register();
9377}
9378
9379Register fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9380 if (RetVT.SimpleTy != MVT::nxv8i1)
9381 return Register();
9382 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9383 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9384 }
9385 return Register();
9386}
9387
9388Register fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9389 if (RetVT.SimpleTy != MVT::nxv16i1)
9390 return Register();
9391 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9392 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9393 }
9394 return Register();
9395}
9396
9397Register fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9398 if (RetVT.SimpleTy != MVT::nxv16i8)
9399 return Register();
9400 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9401 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9402 }
9403 return Register();
9404}
9405
9406Register fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9407 if (RetVT.SimpleTy != MVT::nxv8i16)
9408 return Register();
9409 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9410 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9411 }
9412 return Register();
9413}
9414
9415Register fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9416 if (RetVT.SimpleTy != MVT::nxv4i32)
9417 return Register();
9418 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9419 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9420 }
9421 return Register();
9422}
9423
9424Register fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9425 if (RetVT.SimpleTy != MVT::nxv2i64)
9426 return Register();
9427 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9428 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9429 }
9430 return Register();
9431}
9432
9433Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9434 if (RetVT.SimpleTy != MVT::nxv2f16)
9435 return Register();
9436 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9437 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9438 }
9439 return Register();
9440}
9441
9442Register fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9443 if (RetVT.SimpleTy != MVT::nxv4f16)
9444 return Register();
9445 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9446 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9447 }
9448 return Register();
9449}
9450
9451Register fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9452 if (RetVT.SimpleTy != MVT::nxv8f16)
9453 return Register();
9454 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9455 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9456 }
9457 return Register();
9458}
9459
9460Register fastEmit_AArch64ISD_UZP2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9461 if (RetVT.SimpleTy != MVT::nxv2bf16)
9462 return Register();
9463 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9464 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9465 }
9466 return Register();
9467}
9468
9469Register fastEmit_AArch64ISD_UZP2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9470 if (RetVT.SimpleTy != MVT::nxv4bf16)
9471 return Register();
9472 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9473 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9474 }
9475 return Register();
9476}
9477
9478Register fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9479 if (RetVT.SimpleTy != MVT::nxv8bf16)
9480 return Register();
9481 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9482 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9483 }
9484 return Register();
9485}
9486
9487Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9488 if (RetVT.SimpleTy != MVT::nxv2f32)
9489 return Register();
9490 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9491 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9492 }
9493 return Register();
9494}
9495
9496Register fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9497 if (RetVT.SimpleTy != MVT::nxv4f32)
9498 return Register();
9499 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9500 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9501 }
9502 return Register();
9503}
9504
9505Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9506 if (RetVT.SimpleTy != MVT::nxv2f64)
9507 return Register();
9508 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9509 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9510 }
9511 return Register();
9512}
9513
9514Register fastEmit_AArch64ISD_UZP2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9515 switch (VT.SimpleTy) {
9516 case MVT::v8i8: return fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(RetVT, Op0, Op1);
9517 case MVT::v16i8: return fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(RetVT, Op0, Op1);
9518 case MVT::v4i16: return fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(RetVT, Op0, Op1);
9519 case MVT::v8i16: return fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(RetVT, Op0, Op1);
9520 case MVT::v2i32: return fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(RetVT, Op0, Op1);
9521 case MVT::v4i32: return fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(RetVT, Op0, Op1);
9522 case MVT::v2i64: return fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(RetVT, Op0, Op1);
9523 case MVT::v4f16: return fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(RetVT, Op0, Op1);
9524 case MVT::v8f16: return fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(RetVT, Op0, Op1);
9525 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(RetVT, Op0, Op1);
9526 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(RetVT, Op0, Op1);
9527 case MVT::v2f32: return fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(RetVT, Op0, Op1);
9528 case MVT::v4f32: return fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(RetVT, Op0, Op1);
9529 case MVT::v2f64: return fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(RetVT, Op0, Op1);
9530 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9531 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9532 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9533 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9534 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9535 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9536 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9537 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9538 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9539 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9540 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9541 case MVT::nxv2bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9542 case MVT::nxv4bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9543 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9544 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9545 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9546 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9547 default: return Register();
9548 }
9549}
9550
9551// FastEmit functions for AArch64ISD::ZIP1.
9552
9553Register fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9554 if (RetVT.SimpleTy != MVT::v8i8)
9555 return Register();
9556 if ((Subtarget->isNeonAvailable())) {
9557 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9558 }
9559 return Register();
9560}
9561
9562Register fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9563 if (RetVT.SimpleTy != MVT::v16i8)
9564 return Register();
9565 if ((Subtarget->isNeonAvailable())) {
9566 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9567 }
9568 return Register();
9569}
9570
9571Register fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9572 if (RetVT.SimpleTy != MVT::v4i16)
9573 return Register();
9574 if ((Subtarget->isNeonAvailable())) {
9575 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9576 }
9577 return Register();
9578}
9579
9580Register fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9581 if (RetVT.SimpleTy != MVT::v8i16)
9582 return Register();
9583 if ((Subtarget->isNeonAvailable())) {
9584 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9585 }
9586 return Register();
9587}
9588
9589Register fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9590 if (RetVT.SimpleTy != MVT::v2i32)
9591 return Register();
9592 if ((Subtarget->isNeonAvailable())) {
9593 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9594 }
9595 return Register();
9596}
9597
9598Register fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9599 if (RetVT.SimpleTy != MVT::v4i32)
9600 return Register();
9601 if ((Subtarget->isNeonAvailable())) {
9602 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9603 }
9604 return Register();
9605}
9606
9607Register fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9608 if (RetVT.SimpleTy != MVT::v2i64)
9609 return Register();
9610 if ((Subtarget->isNeonAvailable())) {
9611 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9612 }
9613 return Register();
9614}
9615
9616Register fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9617 if (RetVT.SimpleTy != MVT::v4f16)
9618 return Register();
9619 if ((Subtarget->isNeonAvailable())) {
9620 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9621 }
9622 return Register();
9623}
9624
9625Register fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9626 if (RetVT.SimpleTy != MVT::v8f16)
9627 return Register();
9628 if ((Subtarget->isNeonAvailable())) {
9629 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9630 }
9631 return Register();
9632}
9633
9634Register fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9635 if (RetVT.SimpleTy != MVT::v4bf16)
9636 return Register();
9637 if ((Subtarget->isNeonAvailable())) {
9638 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9639 }
9640 return Register();
9641}
9642
9643Register fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9644 if (RetVT.SimpleTy != MVT::v8bf16)
9645 return Register();
9646 if ((Subtarget->isNeonAvailable())) {
9647 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9648 }
9649 return Register();
9650}
9651
9652Register fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9653 if (RetVT.SimpleTy != MVT::v2f32)
9654 return Register();
9655 if ((Subtarget->isNeonAvailable())) {
9656 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9657 }
9658 return Register();
9659}
9660
9661Register fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9662 if (RetVT.SimpleTy != MVT::v4f32)
9663 return Register();
9664 if ((Subtarget->isNeonAvailable())) {
9665 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9666 }
9667 return Register();
9668}
9669
9670Register fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9671 if (RetVT.SimpleTy != MVT::v2f64)
9672 return Register();
9673 if ((Subtarget->isNeonAvailable())) {
9674 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9675 }
9676 return Register();
9677}
9678
9679Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9680 if (RetVT.SimpleTy != MVT::nxv2i1)
9681 return Register();
9682 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9683 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9684 }
9685 return Register();
9686}
9687
9688Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9689 if (RetVT.SimpleTy != MVT::nxv4i1)
9690 return Register();
9691 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9692 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9693 }
9694 return Register();
9695}
9696
9697Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9698 if (RetVT.SimpleTy != MVT::nxv8i1)
9699 return Register();
9700 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9701 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9702 }
9703 return Register();
9704}
9705
9706Register fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9707 if (RetVT.SimpleTy != MVT::nxv16i1)
9708 return Register();
9709 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9710 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9711 }
9712 return Register();
9713}
9714
9715Register fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9716 if (RetVT.SimpleTy != MVT::nxv16i8)
9717 return Register();
9718 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9719 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9720 }
9721 return Register();
9722}
9723
9724Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9725 if (RetVT.SimpleTy != MVT::nxv8i16)
9726 return Register();
9727 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9728 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9729 }
9730 return Register();
9731}
9732
9733Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9734 if (RetVT.SimpleTy != MVT::nxv4i32)
9735 return Register();
9736 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9737 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9738 }
9739 return Register();
9740}
9741
9742Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9743 if (RetVT.SimpleTy != MVT::nxv2i64)
9744 return Register();
9745 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9746 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9747 }
9748 return Register();
9749}
9750
9751Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9752 if (RetVT.SimpleTy != MVT::nxv2f16)
9753 return Register();
9754 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9755 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9756 }
9757 return Register();
9758}
9759
9760Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9761 if (RetVT.SimpleTy != MVT::nxv4f16)
9762 return Register();
9763 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9764 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9765 }
9766 return Register();
9767}
9768
9769Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9770 if (RetVT.SimpleTy != MVT::nxv8f16)
9771 return Register();
9772 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9773 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9774 }
9775 return Register();
9776}
9777
9778Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9779 if (RetVT.SimpleTy != MVT::nxv2bf16)
9780 return Register();
9781 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9782 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9783 }
9784 return Register();
9785}
9786
9787Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9788 if (RetVT.SimpleTy != MVT::nxv4bf16)
9789 return Register();
9790 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9791 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9792 }
9793 return Register();
9794}
9795
9796Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9797 if (RetVT.SimpleTy != MVT::nxv8bf16)
9798 return Register();
9799 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9800 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9801 }
9802 return Register();
9803}
9804
9805Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9806 if (RetVT.SimpleTy != MVT::nxv2f32)
9807 return Register();
9808 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9809 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9810 }
9811 return Register();
9812}
9813
9814Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9815 if (RetVT.SimpleTy != MVT::nxv4f32)
9816 return Register();
9817 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9818 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9819 }
9820 return Register();
9821}
9822
9823Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9824 if (RetVT.SimpleTy != MVT::nxv2f64)
9825 return Register();
9826 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9827 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9828 }
9829 return Register();
9830}
9831
9832Register fastEmit_AArch64ISD_ZIP1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9833 switch (VT.SimpleTy) {
9834 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(RetVT, Op0, Op1);
9835 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(RetVT, Op0, Op1);
9836 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(RetVT, Op0, Op1);
9837 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(RetVT, Op0, Op1);
9838 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(RetVT, Op0, Op1);
9839 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(RetVT, Op0, Op1);
9840 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(RetVT, Op0, Op1);
9841 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(RetVT, Op0, Op1);
9842 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(RetVT, Op0, Op1);
9843 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(RetVT, Op0, Op1);
9844 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(RetVT, Op0, Op1);
9845 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(RetVT, Op0, Op1);
9846 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(RetVT, Op0, Op1);
9847 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(RetVT, Op0, Op1);
9848 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9849 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9850 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9851 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9852 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9853 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9854 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9855 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9856 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9857 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9858 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9859 case MVT::nxv2bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9860 case MVT::nxv4bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9861 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9862 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9863 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9864 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9865 default: return Register();
9866 }
9867}
9868
9869// FastEmit functions for AArch64ISD::ZIP2.
9870
9871Register fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9872 if (RetVT.SimpleTy != MVT::v8i8)
9873 return Register();
9874 if ((Subtarget->isNeonAvailable())) {
9875 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9876 }
9877 return Register();
9878}
9879
9880Register fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9881 if (RetVT.SimpleTy != MVT::v16i8)
9882 return Register();
9883 if ((Subtarget->isNeonAvailable())) {
9884 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9885 }
9886 return Register();
9887}
9888
9889Register fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9890 if (RetVT.SimpleTy != MVT::v4i16)
9891 return Register();
9892 if ((Subtarget->isNeonAvailable())) {
9893 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9894 }
9895 return Register();
9896}
9897
9898Register fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9899 if (RetVT.SimpleTy != MVT::v8i16)
9900 return Register();
9901 if ((Subtarget->isNeonAvailable())) {
9902 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9903 }
9904 return Register();
9905}
9906
9907Register fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9908 if (RetVT.SimpleTy != MVT::v2i32)
9909 return Register();
9910 if ((Subtarget->isNeonAvailable())) {
9911 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9912 }
9913 return Register();
9914}
9915
9916Register fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9917 if (RetVT.SimpleTy != MVT::v4i32)
9918 return Register();
9919 if ((Subtarget->isNeonAvailable())) {
9920 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9921 }
9922 return Register();
9923}
9924
9925Register fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9926 if (RetVT.SimpleTy != MVT::v2i64)
9927 return Register();
9928 if ((Subtarget->isNeonAvailable())) {
9929 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9930 }
9931 return Register();
9932}
9933
9934Register fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9935 if (RetVT.SimpleTy != MVT::v4f16)
9936 return Register();
9937 if ((Subtarget->isNeonAvailable())) {
9938 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9939 }
9940 return Register();
9941}
9942
9943Register fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9944 if (RetVT.SimpleTy != MVT::v8f16)
9945 return Register();
9946 if ((Subtarget->isNeonAvailable())) {
9947 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9948 }
9949 return Register();
9950}
9951
9952Register fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9953 if (RetVT.SimpleTy != MVT::v4bf16)
9954 return Register();
9955 if ((Subtarget->isNeonAvailable())) {
9956 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9957 }
9958 return Register();
9959}
9960
9961Register fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9962 if (RetVT.SimpleTy != MVT::v8bf16)
9963 return Register();
9964 if ((Subtarget->isNeonAvailable())) {
9965 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9966 }
9967 return Register();
9968}
9969
9970Register fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9971 if (RetVT.SimpleTy != MVT::v2f32)
9972 return Register();
9973 if ((Subtarget->isNeonAvailable())) {
9974 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9975 }
9976 return Register();
9977}
9978
9979Register fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9980 if (RetVT.SimpleTy != MVT::v4f32)
9981 return Register();
9982 if ((Subtarget->isNeonAvailable())) {
9983 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9984 }
9985 return Register();
9986}
9987
9988Register fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9989 if (RetVT.SimpleTy != MVT::v2f64)
9990 return Register();
9991 if ((Subtarget->isNeonAvailable())) {
9992 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9993 }
9994 return Register();
9995}
9996
9997Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9998 if (RetVT.SimpleTy != MVT::nxv2i1)
9999 return Register();
10000 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10001 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
10002 }
10003 return Register();
10004}
10005
10006Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
10007 if (RetVT.SimpleTy != MVT::nxv4i1)
10008 return Register();
10009 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10010 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
10011 }
10012 return Register();
10013}
10014
10015Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
10016 if (RetVT.SimpleTy != MVT::nxv8i1)
10017 return Register();
10018 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10019 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
10020 }
10021 return Register();
10022}
10023
10024Register fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
10025 if (RetVT.SimpleTy != MVT::nxv16i1)
10026 return Register();
10027 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10028 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
10029 }
10030 return Register();
10031}
10032
10033Register fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10034 if (RetVT.SimpleTy != MVT::nxv16i8)
10035 return Register();
10036 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10037 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
10038 }
10039 return Register();
10040}
10041
10042Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10043 if (RetVT.SimpleTy != MVT::nxv8i16)
10044 return Register();
10045 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10046 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10047 }
10048 return Register();
10049}
10050
10051Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10052 if (RetVT.SimpleTy != MVT::nxv4i32)
10053 return Register();
10054 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10055 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10056 }
10057 return Register();
10058}
10059
10060Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10061 if (RetVT.SimpleTy != MVT::nxv2i64)
10062 return Register();
10063 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10064 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10065 }
10066 return Register();
10067}
10068
10069Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
10070 if (RetVT.SimpleTy != MVT::nxv2f16)
10071 return Register();
10072 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10073 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10074 }
10075 return Register();
10076}
10077
10078Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10079 if (RetVT.SimpleTy != MVT::nxv4f16)
10080 return Register();
10081 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10082 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10083 }
10084 return Register();
10085}
10086
10087Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10088 if (RetVT.SimpleTy != MVT::nxv8f16)
10089 return Register();
10090 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10091 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10092 }
10093 return Register();
10094}
10095
10096Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10097 if (RetVT.SimpleTy != MVT::nxv2bf16)
10098 return Register();
10099 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10100 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10101 }
10102 return Register();
10103}
10104
10105Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10106 if (RetVT.SimpleTy != MVT::nxv4bf16)
10107 return Register();
10108 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10109 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10110 }
10111 return Register();
10112}
10113
10114Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10115 if (RetVT.SimpleTy != MVT::nxv8bf16)
10116 return Register();
10117 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10118 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10119 }
10120 return Register();
10121}
10122
10123Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10124 if (RetVT.SimpleTy != MVT::nxv2f32)
10125 return Register();
10126 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10127 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10128 }
10129 return Register();
10130}
10131
10132Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10133 if (RetVT.SimpleTy != MVT::nxv4f32)
10134 return Register();
10135 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10136 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10137 }
10138 return Register();
10139}
10140
10141Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10142 if (RetVT.SimpleTy != MVT::nxv2f64)
10143 return Register();
10144 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10145 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10146 }
10147 return Register();
10148}
10149
10150Register fastEmit_AArch64ISD_ZIP2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10151 switch (VT.SimpleTy) {
10152 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(RetVT, Op0, Op1);
10153 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(RetVT, Op0, Op1);
10154 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(RetVT, Op0, Op1);
10155 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(RetVT, Op0, Op1);
10156 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(RetVT, Op0, Op1);
10157 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(RetVT, Op0, Op1);
10158 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(RetVT, Op0, Op1);
10159 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(RetVT, Op0, Op1);
10160 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(RetVT, Op0, Op1);
10161 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(RetVT, Op0, Op1);
10162 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(RetVT, Op0, Op1);
10163 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(RetVT, Op0, Op1);
10164 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(RetVT, Op0, Op1);
10165 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(RetVT, Op0, Op1);
10166 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
10167 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
10168 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
10169 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
10170 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10171 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10172 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10173 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10174 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
10175 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
10176 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
10177 case MVT::nxv2bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
10178 case MVT::nxv4bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
10179 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
10180 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
10181 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
10182 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
10183 default: return Register();
10184 }
10185}
10186
10187// FastEmit functions for ISD::ABDS.
10188
10189Register fastEmit_ISD_ABDS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10190 if (RetVT.SimpleTy != MVT::v8i8)
10191 return Register();
10192 if ((Subtarget->isNeonAvailable())) {
10193 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10194 }
10195 return Register();
10196}
10197
10198Register fastEmit_ISD_ABDS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10199 if (RetVT.SimpleTy != MVT::v16i8)
10200 return Register();
10201 if ((Subtarget->isNeonAvailable())) {
10202 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10203 }
10204 return Register();
10205}
10206
10207Register fastEmit_ISD_ABDS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10208 if (RetVT.SimpleTy != MVT::v4i16)
10209 return Register();
10210 if ((Subtarget->isNeonAvailable())) {
10211 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10212 }
10213 return Register();
10214}
10215
10216Register fastEmit_ISD_ABDS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10217 if (RetVT.SimpleTy != MVT::v8i16)
10218 return Register();
10219 if ((Subtarget->isNeonAvailable())) {
10220 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10221 }
10222 return Register();
10223}
10224
10225Register fastEmit_ISD_ABDS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10226 if (RetVT.SimpleTy != MVT::v2i32)
10227 return Register();
10228 if ((Subtarget->isNeonAvailable())) {
10229 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10230 }
10231 return Register();
10232}
10233
10234Register fastEmit_ISD_ABDS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10235 if (RetVT.SimpleTy != MVT::v4i32)
10236 return Register();
10237 if ((Subtarget->isNeonAvailable())) {
10238 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10239 }
10240 return Register();
10241}
10242
10243Register fastEmit_ISD_ABDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10244 switch (VT.SimpleTy) {
10245 case MVT::v8i8: return fastEmit_ISD_ABDS_MVT_v8i8_rr(RetVT, Op0, Op1);
10246 case MVT::v16i8: return fastEmit_ISD_ABDS_MVT_v16i8_rr(RetVT, Op0, Op1);
10247 case MVT::v4i16: return fastEmit_ISD_ABDS_MVT_v4i16_rr(RetVT, Op0, Op1);
10248 case MVT::v8i16: return fastEmit_ISD_ABDS_MVT_v8i16_rr(RetVT, Op0, Op1);
10249 case MVT::v2i32: return fastEmit_ISD_ABDS_MVT_v2i32_rr(RetVT, Op0, Op1);
10250 case MVT::v4i32: return fastEmit_ISD_ABDS_MVT_v4i32_rr(RetVT, Op0, Op1);
10251 default: return Register();
10252 }
10253}
10254
10255// FastEmit functions for ISD::ABDU.
10256
10257Register fastEmit_ISD_ABDU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10258 if (RetVT.SimpleTy != MVT::v8i8)
10259 return Register();
10260 if ((Subtarget->isNeonAvailable())) {
10261 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10262 }
10263 return Register();
10264}
10265
10266Register fastEmit_ISD_ABDU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10267 if (RetVT.SimpleTy != MVT::v16i8)
10268 return Register();
10269 if ((Subtarget->isNeonAvailable())) {
10270 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10271 }
10272 return Register();
10273}
10274
10275Register fastEmit_ISD_ABDU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10276 if (RetVT.SimpleTy != MVT::v4i16)
10277 return Register();
10278 if ((Subtarget->isNeonAvailable())) {
10279 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10280 }
10281 return Register();
10282}
10283
10284Register fastEmit_ISD_ABDU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10285 if (RetVT.SimpleTy != MVT::v8i16)
10286 return Register();
10287 if ((Subtarget->isNeonAvailable())) {
10288 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10289 }
10290 return Register();
10291}
10292
10293Register fastEmit_ISD_ABDU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10294 if (RetVT.SimpleTy != MVT::v2i32)
10295 return Register();
10296 if ((Subtarget->isNeonAvailable())) {
10297 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10298 }
10299 return Register();
10300}
10301
10302Register fastEmit_ISD_ABDU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10303 if (RetVT.SimpleTy != MVT::v4i32)
10304 return Register();
10305 if ((Subtarget->isNeonAvailable())) {
10306 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10307 }
10308 return Register();
10309}
10310
10311Register fastEmit_ISD_ABDU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10312 switch (VT.SimpleTy) {
10313 case MVT::v8i8: return fastEmit_ISD_ABDU_MVT_v8i8_rr(RetVT, Op0, Op1);
10314 case MVT::v16i8: return fastEmit_ISD_ABDU_MVT_v16i8_rr(RetVT, Op0, Op1);
10315 case MVT::v4i16: return fastEmit_ISD_ABDU_MVT_v4i16_rr(RetVT, Op0, Op1);
10316 case MVT::v8i16: return fastEmit_ISD_ABDU_MVT_v8i16_rr(RetVT, Op0, Op1);
10317 case MVT::v2i32: return fastEmit_ISD_ABDU_MVT_v2i32_rr(RetVT, Op0, Op1);
10318 case MVT::v4i32: return fastEmit_ISD_ABDU_MVT_v4i32_rr(RetVT, Op0, Op1);
10319 default: return Register();
10320 }
10321}
10322
10323// FastEmit functions for ISD::ADD.
10324
10325Register fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10326 if (RetVT.SimpleTy != MVT::i32)
10327 return Register();
10328 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
10329}
10330
10331Register fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10332 if (RetVT.SimpleTy != MVT::i64)
10333 return Register();
10334 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
10335}
10336
10337Register fastEmit_ISD_ADD_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10338 if (RetVT.SimpleTy != MVT::v8i8)
10339 return Register();
10340 if ((Subtarget->isNeonAvailable())) {
10341 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10342 }
10343 return Register();
10344}
10345
10346Register fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10347 if (RetVT.SimpleTy != MVT::v16i8)
10348 return Register();
10349 if ((Subtarget->isNeonAvailable())) {
10350 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10351 }
10352 return Register();
10353}
10354
10355Register fastEmit_ISD_ADD_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10356 if (RetVT.SimpleTy != MVT::v4i16)
10357 return Register();
10358 if ((Subtarget->isNeonAvailable())) {
10359 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10360 }
10361 return Register();
10362}
10363
10364Register fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10365 if (RetVT.SimpleTy != MVT::v8i16)
10366 return Register();
10367 if ((Subtarget->isNeonAvailable())) {
10368 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10369 }
10370 return Register();
10371}
10372
10373Register fastEmit_ISD_ADD_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10374 if (RetVT.SimpleTy != MVT::v2i32)
10375 return Register();
10376 if ((Subtarget->isNeonAvailable())) {
10377 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10378 }
10379 return Register();
10380}
10381
10382Register fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10383 if (RetVT.SimpleTy != MVT::v4i32)
10384 return Register();
10385 if ((Subtarget->isNeonAvailable())) {
10386 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10387 }
10388 return Register();
10389}
10390
10391Register fastEmit_ISD_ADD_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
10392 if (RetVT.SimpleTy != MVT::v1i64)
10393 return Register();
10394 if ((Subtarget->isNeonAvailable())) {
10395 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
10396 }
10397 return Register();
10398}
10399
10400Register fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10401 if (RetVT.SimpleTy != MVT::v2i64)
10402 return Register();
10403 if ((Subtarget->isNeonAvailable())) {
10404 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10405 }
10406 return Register();
10407}
10408
10409Register fastEmit_ISD_ADD_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10410 if (RetVT.SimpleTy != MVT::nxv16i8)
10411 return Register();
10412 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10413 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
10414 }
10415 return Register();
10416}
10417
10418Register fastEmit_ISD_ADD_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10419 if (RetVT.SimpleTy != MVT::nxv8i16)
10420 return Register();
10421 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10422 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10423 }
10424 return Register();
10425}
10426
10427Register fastEmit_ISD_ADD_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10428 if (RetVT.SimpleTy != MVT::nxv4i32)
10429 return Register();
10430 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10431 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10432 }
10433 return Register();
10434}
10435
10436Register fastEmit_ISD_ADD_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10437 if (RetVT.SimpleTy != MVT::nxv2i64)
10438 return Register();
10439 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10440 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10441 }
10442 return Register();
10443}
10444
10445Register fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10446 switch (VT.SimpleTy) {
10447 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
10448 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op1);
10449 case MVT::v8i8: return fastEmit_ISD_ADD_MVT_v8i8_rr(RetVT, Op0, Op1);
10450 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
10451 case MVT::v4i16: return fastEmit_ISD_ADD_MVT_v4i16_rr(RetVT, Op0, Op1);
10452 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
10453 case MVT::v2i32: return fastEmit_ISD_ADD_MVT_v2i32_rr(RetVT, Op0, Op1);
10454 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
10455 case MVT::v1i64: return fastEmit_ISD_ADD_MVT_v1i64_rr(RetVT, Op0, Op1);
10456 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
10457 case MVT::nxv16i8: return fastEmit_ISD_ADD_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10458 case MVT::nxv8i16: return fastEmit_ISD_ADD_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10459 case MVT::nxv4i32: return fastEmit_ISD_ADD_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10460 case MVT::nxv2i64: return fastEmit_ISD_ADD_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10461 default: return Register();
10462 }
10463}
10464
10465// FastEmit functions for ISD::AND.
10466
10467Register fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10468 if (RetVT.SimpleTy != MVT::i32)
10469 return Register();
10470 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
10471}
10472
10473Register fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10474 if (RetVT.SimpleTy != MVT::i64)
10475 return Register();
10476 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
10477}
10478
10479Register fastEmit_ISD_AND_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10480 if (RetVT.SimpleTy != MVT::v8i8)
10481 return Register();
10482 if ((Subtarget->isNeonAvailable())) {
10483 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10484 }
10485 return Register();
10486}
10487
10488Register fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10489 if (RetVT.SimpleTy != MVT::v16i8)
10490 return Register();
10491 if ((Subtarget->isNeonAvailable())) {
10492 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10493 }
10494 return Register();
10495}
10496
10497Register fastEmit_ISD_AND_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10498 if (RetVT.SimpleTy != MVT::v4i16)
10499 return Register();
10500 if ((Subtarget->isNeonAvailable())) {
10501 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10502 }
10503 return Register();
10504}
10505
10506Register fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10507 if (RetVT.SimpleTy != MVT::v8i16)
10508 return Register();
10509 if ((Subtarget->isNeonAvailable())) {
10510 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10511 }
10512 return Register();
10513}
10514
10515Register fastEmit_ISD_AND_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10516 if (RetVT.SimpleTy != MVT::v2i32)
10517 return Register();
10518 if ((Subtarget->isNeonAvailable())) {
10519 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10520 }
10521 return Register();
10522}
10523
10524Register fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10525 if (RetVT.SimpleTy != MVT::v4i32)
10526 return Register();
10527 if ((Subtarget->isNeonAvailable())) {
10528 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10529 }
10530 return Register();
10531}
10532
10533Register fastEmit_ISD_AND_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
10534 if (RetVT.SimpleTy != MVT::v1i64)
10535 return Register();
10536 if ((Subtarget->isNeonAvailable())) {
10537 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10538 }
10539 return Register();
10540}
10541
10542Register fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10543 if (RetVT.SimpleTy != MVT::v2i64)
10544 return Register();
10545 if ((Subtarget->isNeonAvailable())) {
10546 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10547 }
10548 return Register();
10549}
10550
10551Register fastEmit_ISD_AND_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10552 if (RetVT.SimpleTy != MVT::nxv16i8)
10553 return Register();
10554 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10555 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10556 }
10557 return Register();
10558}
10559
10560Register fastEmit_ISD_AND_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10561 if (RetVT.SimpleTy != MVT::nxv8i16)
10562 return Register();
10563 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10564 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10565 }
10566 return Register();
10567}
10568
10569Register fastEmit_ISD_AND_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10570 if (RetVT.SimpleTy != MVT::nxv4i32)
10571 return Register();
10572 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10573 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10574 }
10575 return Register();
10576}
10577
10578Register fastEmit_ISD_AND_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10579 if (RetVT.SimpleTy != MVT::nxv2i64)
10580 return Register();
10581 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10582 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10583 }
10584 return Register();
10585}
10586
10587Register fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10588 switch (VT.SimpleTy) {
10589 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
10590 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op1);
10591 case MVT::v8i8: return fastEmit_ISD_AND_MVT_v8i8_rr(RetVT, Op0, Op1);
10592 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
10593 case MVT::v4i16: return fastEmit_ISD_AND_MVT_v4i16_rr(RetVT, Op0, Op1);
10594 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
10595 case MVT::v2i32: return fastEmit_ISD_AND_MVT_v2i32_rr(RetVT, Op0, Op1);
10596 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
10597 case MVT::v1i64: return fastEmit_ISD_AND_MVT_v1i64_rr(RetVT, Op0, Op1);
10598 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
10599 case MVT::nxv16i8: return fastEmit_ISD_AND_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10600 case MVT::nxv8i16: return fastEmit_ISD_AND_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10601 case MVT::nxv4i32: return fastEmit_ISD_AND_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10602 case MVT::nxv2i64: return fastEmit_ISD_AND_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10603 default: return Register();
10604 }
10605}
10606
10607// FastEmit functions for ISD::AVGCEILS.
10608
10609Register fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10610 if (RetVT.SimpleTy != MVT::v8i8)
10611 return Register();
10612 if ((Subtarget->isNeonAvailable())) {
10613 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10614 }
10615 return Register();
10616}
10617
10618Register fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10619 if (RetVT.SimpleTy != MVT::v16i8)
10620 return Register();
10621 if ((Subtarget->isNeonAvailable())) {
10622 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10623 }
10624 return Register();
10625}
10626
10627Register fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10628 if (RetVT.SimpleTy != MVT::v4i16)
10629 return Register();
10630 if ((Subtarget->isNeonAvailable())) {
10631 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10632 }
10633 return Register();
10634}
10635
10636Register fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10637 if (RetVT.SimpleTy != MVT::v8i16)
10638 return Register();
10639 if ((Subtarget->isNeonAvailable())) {
10640 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10641 }
10642 return Register();
10643}
10644
10645Register fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10646 if (RetVT.SimpleTy != MVT::v2i32)
10647 return Register();
10648 if ((Subtarget->isNeonAvailable())) {
10649 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10650 }
10651 return Register();
10652}
10653
10654Register fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10655 if (RetVT.SimpleTy != MVT::v4i32)
10656 return Register();
10657 if ((Subtarget->isNeonAvailable())) {
10658 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10659 }
10660 return Register();
10661}
10662
10663Register fastEmit_ISD_AVGCEILS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10664 switch (VT.SimpleTy) {
10665 case MVT::v8i8: return fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(RetVT, Op0, Op1);
10666 case MVT::v16i8: return fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(RetVT, Op0, Op1);
10667 case MVT::v4i16: return fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(RetVT, Op0, Op1);
10668 case MVT::v8i16: return fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(RetVT, Op0, Op1);
10669 case MVT::v2i32: return fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(RetVT, Op0, Op1);
10670 case MVT::v4i32: return fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(RetVT, Op0, Op1);
10671 default: return Register();
10672 }
10673}
10674
10675// FastEmit functions for ISD::AVGCEILU.
10676
10677Register fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10678 if (RetVT.SimpleTy != MVT::v8i8)
10679 return Register();
10680 if ((Subtarget->isNeonAvailable())) {
10681 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10682 }
10683 return Register();
10684}
10685
10686Register fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10687 if (RetVT.SimpleTy != MVT::v16i8)
10688 return Register();
10689 if ((Subtarget->isNeonAvailable())) {
10690 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10691 }
10692 return Register();
10693}
10694
10695Register fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10696 if (RetVT.SimpleTy != MVT::v4i16)
10697 return Register();
10698 if ((Subtarget->isNeonAvailable())) {
10699 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10700 }
10701 return Register();
10702}
10703
10704Register fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10705 if (RetVT.SimpleTy != MVT::v8i16)
10706 return Register();
10707 if ((Subtarget->isNeonAvailable())) {
10708 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10709 }
10710 return Register();
10711}
10712
10713Register fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10714 if (RetVT.SimpleTy != MVT::v2i32)
10715 return Register();
10716 if ((Subtarget->isNeonAvailable())) {
10717 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10718 }
10719 return Register();
10720}
10721
10722Register fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10723 if (RetVT.SimpleTy != MVT::v4i32)
10724 return Register();
10725 if ((Subtarget->isNeonAvailable())) {
10726 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10727 }
10728 return Register();
10729}
10730
10731Register fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10732 switch (VT.SimpleTy) {
10733 case MVT::v8i8: return fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(RetVT, Op0, Op1);
10734 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
10735 case MVT::v4i16: return fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(RetVT, Op0, Op1);
10736 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
10737 case MVT::v2i32: return fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(RetVT, Op0, Op1);
10738 case MVT::v4i32: return fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(RetVT, Op0, Op1);
10739 default: return Register();
10740 }
10741}
10742
10743// FastEmit functions for ISD::AVGFLOORS.
10744
10745Register fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10746 if (RetVT.SimpleTy != MVT::v8i8)
10747 return Register();
10748 if ((Subtarget->isNeonAvailable())) {
10749 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10750 }
10751 return Register();
10752}
10753
10754Register fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10755 if (RetVT.SimpleTy != MVT::v16i8)
10756 return Register();
10757 if ((Subtarget->isNeonAvailable())) {
10758 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10759 }
10760 return Register();
10761}
10762
10763Register fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10764 if (RetVT.SimpleTy != MVT::v4i16)
10765 return Register();
10766 if ((Subtarget->isNeonAvailable())) {
10767 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10768 }
10769 return Register();
10770}
10771
10772Register fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10773 if (RetVT.SimpleTy != MVT::v8i16)
10774 return Register();
10775 if ((Subtarget->isNeonAvailable())) {
10776 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10777 }
10778 return Register();
10779}
10780
10781Register fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10782 if (RetVT.SimpleTy != MVT::v2i32)
10783 return Register();
10784 if ((Subtarget->isNeonAvailable())) {
10785 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10786 }
10787 return Register();
10788}
10789
10790Register fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10791 if (RetVT.SimpleTy != MVT::v4i32)
10792 return Register();
10793 if ((Subtarget->isNeonAvailable())) {
10794 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10795 }
10796 return Register();
10797}
10798
10799Register fastEmit_ISD_AVGFLOORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10800 switch (VT.SimpleTy) {
10801 case MVT::v8i8: return fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(RetVT, Op0, Op1);
10802 case MVT::v16i8: return fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(RetVT, Op0, Op1);
10803 case MVT::v4i16: return fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(RetVT, Op0, Op1);
10804 case MVT::v8i16: return fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(RetVT, Op0, Op1);
10805 case MVT::v2i32: return fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(RetVT, Op0, Op1);
10806 case MVT::v4i32: return fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(RetVT, Op0, Op1);
10807 default: return Register();
10808 }
10809}
10810
10811// FastEmit functions for ISD::AVGFLOORU.
10812
10813Register fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10814 if (RetVT.SimpleTy != MVT::v8i8)
10815 return Register();
10816 if ((Subtarget->isNeonAvailable())) {
10817 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10818 }
10819 return Register();
10820}
10821
10822Register fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10823 if (RetVT.SimpleTy != MVT::v16i8)
10824 return Register();
10825 if ((Subtarget->isNeonAvailable())) {
10826 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10827 }
10828 return Register();
10829}
10830
10831Register fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10832 if (RetVT.SimpleTy != MVT::v4i16)
10833 return Register();
10834 if ((Subtarget->isNeonAvailable())) {
10835 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10836 }
10837 return Register();
10838}
10839
10840Register fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10841 if (RetVT.SimpleTy != MVT::v8i16)
10842 return Register();
10843 if ((Subtarget->isNeonAvailable())) {
10844 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10845 }
10846 return Register();
10847}
10848
10849Register fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10850 if (RetVT.SimpleTy != MVT::v2i32)
10851 return Register();
10852 if ((Subtarget->isNeonAvailable())) {
10853 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10854 }
10855 return Register();
10856}
10857
10858Register fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10859 if (RetVT.SimpleTy != MVT::v4i32)
10860 return Register();
10861 if ((Subtarget->isNeonAvailable())) {
10862 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10863 }
10864 return Register();
10865}
10866
10867Register fastEmit_ISD_AVGFLOORU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10868 switch (VT.SimpleTy) {
10869 case MVT::v8i8: return fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(RetVT, Op0, Op1);
10870 case MVT::v16i8: return fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(RetVT, Op0, Op1);
10871 case MVT::v4i16: return fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(RetVT, Op0, Op1);
10872 case MVT::v8i16: return fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(RetVT, Op0, Op1);
10873 case MVT::v2i32: return fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(RetVT, Op0, Op1);
10874 case MVT::v4i32: return fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(RetVT, Op0, Op1);
10875 default: return Register();
10876 }
10877}
10878
10879// FastEmit functions for ISD::CLMUL.
10880
10881Register fastEmit_ISD_CLMUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10882 if (RetVT.SimpleTy != MVT::v8i8)
10883 return Register();
10884 if ((Subtarget->isNeonAvailable())) {
10885 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10886 }
10887 return Register();
10888}
10889
10890Register fastEmit_ISD_CLMUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10891 if (RetVT.SimpleTy != MVT::v16i8)
10892 return Register();
10893 if ((Subtarget->isNeonAvailable())) {
10894 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10895 }
10896 return Register();
10897}
10898
10899Register fastEmit_ISD_CLMUL_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10900 if (RetVT.SimpleTy != MVT::nxv16i8)
10901 return Register();
10902 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2() || Subtarget->hasSME()))) {
10903 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMUL_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
10904 }
10905 return Register();
10906}
10907
10908Register fastEmit_ISD_CLMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10909 switch (VT.SimpleTy) {
10910 case MVT::v8i8: return fastEmit_ISD_CLMUL_MVT_v8i8_rr(RetVT, Op0, Op1);
10911 case MVT::v16i8: return fastEmit_ISD_CLMUL_MVT_v16i8_rr(RetVT, Op0, Op1);
10912 case MVT::nxv16i8: return fastEmit_ISD_CLMUL_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10913 default: return Register();
10914 }
10915}
10916
10917// FastEmit functions for ISD::CONCAT_VECTORS.
10918
10919Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(MVT RetVT, Register Op0, Register Op1) {
10920 if (RetVT.SimpleTy != MVT::nxv2i1)
10921 return Register();
10922 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10923 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
10924 }
10925 return Register();
10926}
10927
10928Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
10929 if (RetVT.SimpleTy != MVT::nxv4i1)
10930 return Register();
10931 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10932 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
10933 }
10934 return Register();
10935}
10936
10937Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
10938 if (RetVT.SimpleTy != MVT::nxv8i1)
10939 return Register();
10940 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10941 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
10942 }
10943 return Register();
10944}
10945
10946Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
10947 if (RetVT.SimpleTy != MVT::nxv16i1)
10948 return Register();
10949 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10950 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
10951 }
10952 return Register();
10953}
10954
10955Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
10956 if (RetVT.SimpleTy != MVT::nxv4f16)
10957 return Register();
10958 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10959 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10960 }
10961 return Register();
10962}
10963
10964Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10965 if (RetVT.SimpleTy != MVT::nxv8f16)
10966 return Register();
10967 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10968 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10969 }
10970 return Register();
10971}
10972
10973Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10974 if (RetVT.SimpleTy != MVT::nxv4bf16)
10975 return Register();
10976 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10977 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10978 }
10979 return Register();
10980}
10981
10982Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10983 if (RetVT.SimpleTy != MVT::nxv8bf16)
10984 return Register();
10985 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10986 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10987 }
10988 return Register();
10989}
10990
10991Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10992 if (RetVT.SimpleTy != MVT::nxv4f32)
10993 return Register();
10994 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10995 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10996 }
10997 return Register();
10998}
10999
11000Register fastEmit_ISD_CONCAT_VECTORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11001 switch (VT.SimpleTy) {
11002 case MVT::nxv1i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(RetVT, Op0, Op1);
11003 case MVT::nxv2i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(RetVT, Op0, Op1);
11004 case MVT::nxv4i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(RetVT, Op0, Op1);
11005 case MVT::nxv8i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(RetVT, Op0, Op1);
11006 case MVT::nxv2f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(RetVT, Op0, Op1);
11007 case MVT::nxv4f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(RetVT, Op0, Op1);
11008 case MVT::nxv2bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
11009 case MVT::nxv4bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
11010 case MVT::nxv2f32: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(RetVT, Op0, Op1);
11011 default: return Register();
11012 }
11013}
11014
11015// FastEmit functions for ISD::FADD.
11016
11017Register fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11018 if (RetVT.SimpleTy != MVT::f16)
11019 return Register();
11020 if ((Subtarget->hasFullFP16())) {
11021 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11022 }
11023 return Register();
11024}
11025
11026Register fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11027 if (RetVT.SimpleTy != MVT::f32)
11028 return Register();
11029 if ((Subtarget->hasFPARMv8())) {
11030 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11031 }
11032 return Register();
11033}
11034
11035Register fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11036 if (RetVT.SimpleTy != MVT::f64)
11037 return Register();
11038 if ((Subtarget->hasFPARMv8())) {
11039 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11040 }
11041 return Register();
11042}
11043
11044Register fastEmit_ISD_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11045 if (RetVT.SimpleTy != MVT::v4f16)
11046 return Register();
11047 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11048 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11049 }
11050 return Register();
11051}
11052
11053Register fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11054 if (RetVT.SimpleTy != MVT::v8f16)
11055 return Register();
11056 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11057 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11058 }
11059 return Register();
11060}
11061
11062Register fastEmit_ISD_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11063 if (RetVT.SimpleTy != MVT::v2f32)
11064 return Register();
11065 if ((Subtarget->isNeonAvailable())) {
11066 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11067 }
11068 return Register();
11069}
11070
11071Register fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11072 if (RetVT.SimpleTy != MVT::v4f32)
11073 return Register();
11074 if ((Subtarget->isNeonAvailable())) {
11075 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11076 }
11077 return Register();
11078}
11079
11080Register fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11081 if (RetVT.SimpleTy != MVT::v2f64)
11082 return Register();
11083 if ((Subtarget->isNeonAvailable())) {
11084 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11085 }
11086 return Register();
11087}
11088
11089Register fastEmit_ISD_FADD_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11090 if (RetVT.SimpleTy != MVT::nxv8f16)
11091 return Register();
11092 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11093 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11094 }
11095 return Register();
11096}
11097
11098Register fastEmit_ISD_FADD_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11099 if (RetVT.SimpleTy != MVT::nxv8bf16)
11100 return Register();
11101 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
11102 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFADD_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11103 }
11104 return Register();
11105}
11106
11107Register fastEmit_ISD_FADD_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11108 if (RetVT.SimpleTy != MVT::nxv4f32)
11109 return Register();
11110 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11111 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11112 }
11113 return Register();
11114}
11115
11116Register fastEmit_ISD_FADD_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11117 if (RetVT.SimpleTy != MVT::nxv2f64)
11118 return Register();
11119 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11120 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11121 }
11122 return Register();
11123}
11124
11125Register fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11126 switch (VT.SimpleTy) {
11127 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
11128 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
11129 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
11130 case MVT::v4f16: return fastEmit_ISD_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
11131 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
11132 case MVT::v2f32: return fastEmit_ISD_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
11133 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
11134 case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
11135 case MVT::nxv8f16: return fastEmit_ISD_FADD_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11136 case MVT::nxv8bf16: return fastEmit_ISD_FADD_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11137 case MVT::nxv4f32: return fastEmit_ISD_FADD_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11138 case MVT::nxv2f64: return fastEmit_ISD_FADD_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11139 default: return Register();
11140 }
11141}
11142
11143// FastEmit functions for ISD::FDIV.
11144
11145Register fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11146 if (RetVT.SimpleTy != MVT::f16)
11147 return Register();
11148 if ((Subtarget->hasFullFP16())) {
11149 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11150 }
11151 return Register();
11152}
11153
11154Register fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11155 if (RetVT.SimpleTy != MVT::f32)
11156 return Register();
11157 if ((Subtarget->hasFPARMv8())) {
11158 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11159 }
11160 return Register();
11161}
11162
11163Register fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11164 if (RetVT.SimpleTy != MVT::f64)
11165 return Register();
11166 if ((Subtarget->hasFPARMv8())) {
11167 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11168 }
11169 return Register();
11170}
11171
11172Register fastEmit_ISD_FDIV_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11173 if (RetVT.SimpleTy != MVT::v4f16)
11174 return Register();
11175 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11176 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11177 }
11178 return Register();
11179}
11180
11181Register fastEmit_ISD_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11182 if (RetVT.SimpleTy != MVT::v8f16)
11183 return Register();
11184 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11185 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11186 }
11187 return Register();
11188}
11189
11190Register fastEmit_ISD_FDIV_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11191 if (RetVT.SimpleTy != MVT::v2f32)
11192 return Register();
11193 if ((Subtarget->isNeonAvailable())) {
11194 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11195 }
11196 return Register();
11197}
11198
11199Register fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11200 if (RetVT.SimpleTy != MVT::v4f32)
11201 return Register();
11202 if ((Subtarget->isNeonAvailable())) {
11203 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11204 }
11205 return Register();
11206}
11207
11208Register fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11209 if (RetVT.SimpleTy != MVT::v2f64)
11210 return Register();
11211 if ((Subtarget->isNeonAvailable())) {
11212 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11213 }
11214 return Register();
11215}
11216
11217Register fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11218 switch (VT.SimpleTy) {
11219 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
11220 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
11221 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
11222 case MVT::v4f16: return fastEmit_ISD_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1);
11223 case MVT::v8f16: return fastEmit_ISD_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
11224 case MVT::v2f32: return fastEmit_ISD_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1);
11225 case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
11226 case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
11227 default: return Register();
11228 }
11229}
11230
11231// FastEmit functions for ISD::FMAXIMUM.
11232
11233Register fastEmit_ISD_FMAXIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11234 if (RetVT.SimpleTy != MVT::f16)
11235 return Register();
11236 if ((Subtarget->hasFullFP16())) {
11237 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11238 }
11239 return Register();
11240}
11241
11242Register fastEmit_ISD_FMAXIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11243 if (RetVT.SimpleTy != MVT::f32)
11244 return Register();
11245 if ((Subtarget->hasFPARMv8())) {
11246 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11247 }
11248 return Register();
11249}
11250
11251Register fastEmit_ISD_FMAXIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11252 if (RetVT.SimpleTy != MVT::f64)
11253 return Register();
11254 if ((Subtarget->hasFPARMv8())) {
11255 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11256 }
11257 return Register();
11258}
11259
11260Register fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11261 if (RetVT.SimpleTy != MVT::v4f16)
11262 return Register();
11263 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11264 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11265 }
11266 return Register();
11267}
11268
11269Register fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11270 if (RetVT.SimpleTy != MVT::v8f16)
11271 return Register();
11272 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11273 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11274 }
11275 return Register();
11276}
11277
11278Register fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11279 if (RetVT.SimpleTy != MVT::v2f32)
11280 return Register();
11281 if ((Subtarget->isNeonAvailable())) {
11282 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11283 }
11284 return Register();
11285}
11286
11287Register fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11288 if (RetVT.SimpleTy != MVT::v4f32)
11289 return Register();
11290 if ((Subtarget->isNeonAvailable())) {
11291 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11292 }
11293 return Register();
11294}
11295
11296Register fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11297 if (RetVT.SimpleTy != MVT::v1f64)
11298 return Register();
11299 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11300}
11301
11302Register fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11303 if (RetVT.SimpleTy != MVT::v2f64)
11304 return Register();
11305 if ((Subtarget->isNeonAvailable())) {
11306 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11307 }
11308 return Register();
11309}
11310
11311Register fastEmit_ISD_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11312 switch (VT.SimpleTy) {
11313 case MVT::f16: return fastEmit_ISD_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1);
11314 case MVT::f32: return fastEmit_ISD_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1);
11315 case MVT::f64: return fastEmit_ISD_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1);
11316 case MVT::v4f16: return fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11317 case MVT::v8f16: return fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11318 case MVT::v2f32: return fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11319 case MVT::v4f32: return fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11320 case MVT::v1f64: return fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11321 case MVT::v2f64: return fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11322 default: return Register();
11323 }
11324}
11325
11326// FastEmit functions for ISD::FMAXNUM.
11327
11328Register fastEmit_ISD_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11329 if (RetVT.SimpleTy != MVT::f16)
11330 return Register();
11331 if ((Subtarget->hasFullFP16())) {
11332 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11333 }
11334 return Register();
11335}
11336
11337Register fastEmit_ISD_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11338 if (RetVT.SimpleTy != MVT::f32)
11339 return Register();
11340 if ((Subtarget->hasFPARMv8())) {
11341 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11342 }
11343 return Register();
11344}
11345
11346Register fastEmit_ISD_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11347 if (RetVT.SimpleTy != MVT::f64)
11348 return Register();
11349 if ((Subtarget->hasFPARMv8())) {
11350 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11351 }
11352 return Register();
11353}
11354
11355Register fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11356 if (RetVT.SimpleTy != MVT::v4f16)
11357 return Register();
11358 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11359 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11360 }
11361 return Register();
11362}
11363
11364Register fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11365 if (RetVT.SimpleTy != MVT::v8f16)
11366 return Register();
11367 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11368 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11369 }
11370 return Register();
11371}
11372
11373Register fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11374 if (RetVT.SimpleTy != MVT::v2f32)
11375 return Register();
11376 if ((Subtarget->isNeonAvailable())) {
11377 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11378 }
11379 return Register();
11380}
11381
11382Register fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11383 if (RetVT.SimpleTy != MVT::v4f32)
11384 return Register();
11385 if ((Subtarget->isNeonAvailable())) {
11386 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11387 }
11388 return Register();
11389}
11390
11391Register fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11392 if (RetVT.SimpleTy != MVT::v1f64)
11393 return Register();
11394 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11395}
11396
11397Register fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11398 if (RetVT.SimpleTy != MVT::v2f64)
11399 return Register();
11400 if ((Subtarget->isNeonAvailable())) {
11401 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11402 }
11403 return Register();
11404}
11405
11406Register fastEmit_ISD_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11407 switch (VT.SimpleTy) {
11408 case MVT::f16: return fastEmit_ISD_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
11409 case MVT::f32: return fastEmit_ISD_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
11410 case MVT::f64: return fastEmit_ISD_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
11411 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11412 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11413 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11414 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11415 case MVT::v1f64: return fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11416 case MVT::v2f64: return fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11417 default: return Register();
11418 }
11419}
11420
11421// FastEmit functions for ISD::FMAXNUM_IEEE.
11422
11423Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11424 if (RetVT.SimpleTy != MVT::f16)
11425 return Register();
11426 if ((Subtarget->hasFullFP16())) {
11427 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11428 }
11429 return Register();
11430}
11431
11432Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11433 if (RetVT.SimpleTy != MVT::f32)
11434 return Register();
11435 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11436}
11437
11438Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11439 if (RetVT.SimpleTy != MVT::f64)
11440 return Register();
11441 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11442}
11443
11444Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11445 if (RetVT.SimpleTy != MVT::v4f16)
11446 return Register();
11447 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11448 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11449 }
11450 return Register();
11451}
11452
11453Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11454 if (RetVT.SimpleTy != MVT::v8f16)
11455 return Register();
11456 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11457 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11458 }
11459 return Register();
11460}
11461
11462Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11463 if (RetVT.SimpleTy != MVT::v2f32)
11464 return Register();
11465 if ((Subtarget->isNeonAvailable())) {
11466 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11467 }
11468 return Register();
11469}
11470
11471Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11472 if (RetVT.SimpleTy != MVT::v4f32)
11473 return Register();
11474 if ((Subtarget->isNeonAvailable())) {
11475 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11476 }
11477 return Register();
11478}
11479
11480Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11481 if (RetVT.SimpleTy != MVT::v2f64)
11482 return Register();
11483 if ((Subtarget->isNeonAvailable())) {
11484 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11485 }
11486 return Register();
11487}
11488
11489Register fastEmit_ISD_FMAXNUM_IEEE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11490 switch (VT.SimpleTy) {
11491 case MVT::f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f16_rr(RetVT, Op0, Op1);
11492 case MVT::f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f32_rr(RetVT, Op0, Op1);
11493 case MVT::f64: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f64_rr(RetVT, Op0, Op1);
11494 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f16_rr(RetVT, Op0, Op1);
11495 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v8f16_rr(RetVT, Op0, Op1);
11496 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f32_rr(RetVT, Op0, Op1);
11497 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f32_rr(RetVT, Op0, Op1);
11498 case MVT::v2f64: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f64_rr(RetVT, Op0, Op1);
11499 default: return Register();
11500 }
11501}
11502
11503// FastEmit functions for ISD::FMINIMUM.
11504
11505Register fastEmit_ISD_FMINIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11506 if (RetVT.SimpleTy != MVT::f16)
11507 return Register();
11508 if ((Subtarget->hasFullFP16())) {
11509 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11510 }
11511 return Register();
11512}
11513
11514Register fastEmit_ISD_FMINIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11515 if (RetVT.SimpleTy != MVT::f32)
11516 return Register();
11517 if ((Subtarget->hasFPARMv8())) {
11518 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11519 }
11520 return Register();
11521}
11522
11523Register fastEmit_ISD_FMINIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11524 if (RetVT.SimpleTy != MVT::f64)
11525 return Register();
11526 if ((Subtarget->hasFPARMv8())) {
11527 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11528 }
11529 return Register();
11530}
11531
11532Register fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11533 if (RetVT.SimpleTy != MVT::v4f16)
11534 return Register();
11535 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11536 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11537 }
11538 return Register();
11539}
11540
11541Register fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11542 if (RetVT.SimpleTy != MVT::v8f16)
11543 return Register();
11544 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11545 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11546 }
11547 return Register();
11548}
11549
11550Register fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11551 if (RetVT.SimpleTy != MVT::v2f32)
11552 return Register();
11553 if ((Subtarget->isNeonAvailable())) {
11554 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11555 }
11556 return Register();
11557}
11558
11559Register fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11560 if (RetVT.SimpleTy != MVT::v4f32)
11561 return Register();
11562 if ((Subtarget->isNeonAvailable())) {
11563 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11564 }
11565 return Register();
11566}
11567
11568Register fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11569 if (RetVT.SimpleTy != MVT::v1f64)
11570 return Register();
11571 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11572}
11573
11574Register fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11575 if (RetVT.SimpleTy != MVT::v2f64)
11576 return Register();
11577 if ((Subtarget->isNeonAvailable())) {
11578 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11579 }
11580 return Register();
11581}
11582
11583Register fastEmit_ISD_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11584 switch (VT.SimpleTy) {
11585 case MVT::f16: return fastEmit_ISD_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1);
11586 case MVT::f32: return fastEmit_ISD_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1);
11587 case MVT::f64: return fastEmit_ISD_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1);
11588 case MVT::v4f16: return fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11589 case MVT::v8f16: return fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11590 case MVT::v2f32: return fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11591 case MVT::v4f32: return fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11592 case MVT::v1f64: return fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11593 case MVT::v2f64: return fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11594 default: return Register();
11595 }
11596}
11597
11598// FastEmit functions for ISD::FMINNUM.
11599
11600Register fastEmit_ISD_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11601 if (RetVT.SimpleTy != MVT::f16)
11602 return Register();
11603 if ((Subtarget->hasFullFP16())) {
11604 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11605 }
11606 return Register();
11607}
11608
11609Register fastEmit_ISD_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11610 if (RetVT.SimpleTy != MVT::f32)
11611 return Register();
11612 if ((Subtarget->hasFPARMv8())) {
11613 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11614 }
11615 return Register();
11616}
11617
11618Register fastEmit_ISD_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11619 if (RetVT.SimpleTy != MVT::f64)
11620 return Register();
11621 if ((Subtarget->hasFPARMv8())) {
11622 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11623 }
11624 return Register();
11625}
11626
11627Register fastEmit_ISD_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11628 if (RetVT.SimpleTy != MVT::v4f16)
11629 return Register();
11630 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11631 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11632 }
11633 return Register();
11634}
11635
11636Register fastEmit_ISD_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11637 if (RetVT.SimpleTy != MVT::v8f16)
11638 return Register();
11639 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11640 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11641 }
11642 return Register();
11643}
11644
11645Register fastEmit_ISD_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11646 if (RetVT.SimpleTy != MVT::v2f32)
11647 return Register();
11648 if ((Subtarget->isNeonAvailable())) {
11649 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11650 }
11651 return Register();
11652}
11653
11654Register fastEmit_ISD_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11655 if (RetVT.SimpleTy != MVT::v4f32)
11656 return Register();
11657 if ((Subtarget->isNeonAvailable())) {
11658 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11659 }
11660 return Register();
11661}
11662
11663Register fastEmit_ISD_FMINNUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11664 if (RetVT.SimpleTy != MVT::v1f64)
11665 return Register();
11666 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11667}
11668
11669Register fastEmit_ISD_FMINNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11670 if (RetVT.SimpleTy != MVT::v2f64)
11671 return Register();
11672 if ((Subtarget->isNeonAvailable())) {
11673 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11674 }
11675 return Register();
11676}
11677
11678Register fastEmit_ISD_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11679 switch (VT.SimpleTy) {
11680 case MVT::f16: return fastEmit_ISD_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
11681 case MVT::f32: return fastEmit_ISD_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
11682 case MVT::f64: return fastEmit_ISD_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
11683 case MVT::v4f16: return fastEmit_ISD_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11684 case MVT::v8f16: return fastEmit_ISD_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11685 case MVT::v2f32: return fastEmit_ISD_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11686 case MVT::v4f32: return fastEmit_ISD_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11687 case MVT::v1f64: return fastEmit_ISD_FMINNUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11688 case MVT::v2f64: return fastEmit_ISD_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11689 default: return Register();
11690 }
11691}
11692
11693// FastEmit functions for ISD::FMINNUM_IEEE.
11694
11695Register fastEmit_ISD_FMINNUM_IEEE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11696 if (RetVT.SimpleTy != MVT::f16)
11697 return Register();
11698 if ((Subtarget->hasFullFP16())) {
11699 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11700 }
11701 return Register();
11702}
11703
11704Register fastEmit_ISD_FMINNUM_IEEE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11705 if (RetVT.SimpleTy != MVT::f32)
11706 return Register();
11707 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11708}
11709
11710Register fastEmit_ISD_FMINNUM_IEEE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11711 if (RetVT.SimpleTy != MVT::f64)
11712 return Register();
11713 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11714}
11715
11716Register fastEmit_ISD_FMINNUM_IEEE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11717 if (RetVT.SimpleTy != MVT::v4f16)
11718 return Register();
11719 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11720 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11721 }
11722 return Register();
11723}
11724
11725Register fastEmit_ISD_FMINNUM_IEEE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11726 if (RetVT.SimpleTy != MVT::v8f16)
11727 return Register();
11728 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11729 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11730 }
11731 return Register();
11732}
11733
11734Register fastEmit_ISD_FMINNUM_IEEE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11735 if (RetVT.SimpleTy != MVT::v2f32)
11736 return Register();
11737 if ((Subtarget->isNeonAvailable())) {
11738 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11739 }
11740 return Register();
11741}
11742
11743Register fastEmit_ISD_FMINNUM_IEEE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11744 if (RetVT.SimpleTy != MVT::v4f32)
11745 return Register();
11746 if ((Subtarget->isNeonAvailable())) {
11747 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11748 }
11749 return Register();
11750}
11751
11752Register fastEmit_ISD_FMINNUM_IEEE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11753 if (RetVT.SimpleTy != MVT::v2f64)
11754 return Register();
11755 if ((Subtarget->isNeonAvailable())) {
11756 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11757 }
11758 return Register();
11759}
11760
11761Register fastEmit_ISD_FMINNUM_IEEE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11762 switch (VT.SimpleTy) {
11763 case MVT::f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_f16_rr(RetVT, Op0, Op1);
11764 case MVT::f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_f32_rr(RetVT, Op0, Op1);
11765 case MVT::f64: return fastEmit_ISD_FMINNUM_IEEE_MVT_f64_rr(RetVT, Op0, Op1);
11766 case MVT::v4f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_v4f16_rr(RetVT, Op0, Op1);
11767 case MVT::v8f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_v8f16_rr(RetVT, Op0, Op1);
11768 case MVT::v2f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_v2f32_rr(RetVT, Op0, Op1);
11769 case MVT::v4f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_v4f32_rr(RetVT, Op0, Op1);
11770 case MVT::v2f64: return fastEmit_ISD_FMINNUM_IEEE_MVT_v2f64_rr(RetVT, Op0, Op1);
11771 default: return Register();
11772 }
11773}
11774
11775// FastEmit functions for ISD::FMUL.
11776
11777Register fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11778 if (RetVT.SimpleTy != MVT::f16)
11779 return Register();
11780 if ((Subtarget->hasFullFP16())) {
11781 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11782 }
11783 return Register();
11784}
11785
11786Register fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11787 if (RetVT.SimpleTy != MVT::f32)
11788 return Register();
11789 if ((Subtarget->hasFPARMv8())) {
11790 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11791 }
11792 return Register();
11793}
11794
11795Register fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11796 if (RetVT.SimpleTy != MVT::f64)
11797 return Register();
11798 if ((Subtarget->hasFPARMv8())) {
11799 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11800 }
11801 return Register();
11802}
11803
11804Register fastEmit_ISD_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11805 if (RetVT.SimpleTy != MVT::v4f16)
11806 return Register();
11807 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11808 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11809 }
11810 return Register();
11811}
11812
11813Register fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11814 if (RetVT.SimpleTy != MVT::v8f16)
11815 return Register();
11816 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11817 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11818 }
11819 return Register();
11820}
11821
11822Register fastEmit_ISD_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11823 if (RetVT.SimpleTy != MVT::v2f32)
11824 return Register();
11825 if ((Subtarget->isNeonAvailable())) {
11826 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11827 }
11828 return Register();
11829}
11830
11831Register fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11832 if (RetVT.SimpleTy != MVT::v4f32)
11833 return Register();
11834 if ((Subtarget->isNeonAvailable())) {
11835 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11836 }
11837 return Register();
11838}
11839
11840Register fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11841 if (RetVT.SimpleTy != MVT::v2f64)
11842 return Register();
11843 if ((Subtarget->isNeonAvailable())) {
11844 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11845 }
11846 return Register();
11847}
11848
11849Register fastEmit_ISD_FMUL_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11850 if (RetVT.SimpleTy != MVT::nxv8f16)
11851 return Register();
11852 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11853 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11854 }
11855 return Register();
11856}
11857
11858Register fastEmit_ISD_FMUL_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11859 if (RetVT.SimpleTy != MVT::nxv8bf16)
11860 return Register();
11861 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
11862 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFMUL_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11863 }
11864 return Register();
11865}
11866
11867Register fastEmit_ISD_FMUL_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11868 if (RetVT.SimpleTy != MVT::nxv4f32)
11869 return Register();
11870 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11871 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11872 }
11873 return Register();
11874}
11875
11876Register fastEmit_ISD_FMUL_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11877 if (RetVT.SimpleTy != MVT::nxv2f64)
11878 return Register();
11879 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11880 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11881 }
11882 return Register();
11883}
11884
11885Register fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11886 switch (VT.SimpleTy) {
11887 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
11888 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
11889 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
11890 case MVT::v4f16: return fastEmit_ISD_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
11891 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
11892 case MVT::v2f32: return fastEmit_ISD_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
11893 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
11894 case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
11895 case MVT::nxv8f16: return fastEmit_ISD_FMUL_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11896 case MVT::nxv8bf16: return fastEmit_ISD_FMUL_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11897 case MVT::nxv4f32: return fastEmit_ISD_FMUL_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11898 case MVT::nxv2f64: return fastEmit_ISD_FMUL_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11899 default: return Register();
11900 }
11901}
11902
11903// FastEmit functions for ISD::FSUB.
11904
11905Register fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11906 if (RetVT.SimpleTy != MVT::f16)
11907 return Register();
11908 if ((Subtarget->hasFullFP16())) {
11909 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11910 }
11911 return Register();
11912}
11913
11914Register fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11915 if (RetVT.SimpleTy != MVT::f32)
11916 return Register();
11917 if ((Subtarget->hasFPARMv8())) {
11918 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11919 }
11920 return Register();
11921}
11922
11923Register fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11924 if (RetVT.SimpleTy != MVT::f64)
11925 return Register();
11926 if ((Subtarget->hasFPARMv8())) {
11927 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11928 }
11929 return Register();
11930}
11931
11932Register fastEmit_ISD_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11933 if (RetVT.SimpleTy != MVT::v4f16)
11934 return Register();
11935 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11936 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11937 }
11938 return Register();
11939}
11940
11941Register fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11942 if (RetVT.SimpleTy != MVT::v8f16)
11943 return Register();
11944 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11945 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11946 }
11947 return Register();
11948}
11949
11950Register fastEmit_ISD_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11951 if (RetVT.SimpleTy != MVT::v2f32)
11952 return Register();
11953 if ((Subtarget->isNeonAvailable())) {
11954 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11955 }
11956 return Register();
11957}
11958
11959Register fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11960 if (RetVT.SimpleTy != MVT::v4f32)
11961 return Register();
11962 if ((Subtarget->isNeonAvailable())) {
11963 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11964 }
11965 return Register();
11966}
11967
11968Register fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11969 if (RetVT.SimpleTy != MVT::v2f64)
11970 return Register();
11971 if ((Subtarget->isNeonAvailable())) {
11972 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11973 }
11974 return Register();
11975}
11976
11977Register fastEmit_ISD_FSUB_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11978 if (RetVT.SimpleTy != MVT::nxv8f16)
11979 return Register();
11980 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11981 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11982 }
11983 return Register();
11984}
11985
11986Register fastEmit_ISD_FSUB_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11987 if (RetVT.SimpleTy != MVT::nxv8bf16)
11988 return Register();
11989 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
11990 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFSUB_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11991 }
11992 return Register();
11993}
11994
11995Register fastEmit_ISD_FSUB_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11996 if (RetVT.SimpleTy != MVT::nxv4f32)
11997 return Register();
11998 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11999 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
12000 }
12001 return Register();
12002}
12003
12004Register fastEmit_ISD_FSUB_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12005 if (RetVT.SimpleTy != MVT::nxv2f64)
12006 return Register();
12007 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12008 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
12009 }
12010 return Register();
12011}
12012
12013Register fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12014 switch (VT.SimpleTy) {
12015 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
12016 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
12017 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
12018 case MVT::v4f16: return fastEmit_ISD_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
12019 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
12020 case MVT::v2f32: return fastEmit_ISD_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
12021 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
12022 case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
12023 case MVT::nxv8f16: return fastEmit_ISD_FSUB_MVT_nxv8f16_rr(RetVT, Op0, Op1);
12024 case MVT::nxv8bf16: return fastEmit_ISD_FSUB_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
12025 case MVT::nxv4f32: return fastEmit_ISD_FSUB_MVT_nxv4f32_rr(RetVT, Op0, Op1);
12026 case MVT::nxv2f64: return fastEmit_ISD_FSUB_MVT_nxv2f64_rr(RetVT, Op0, Op1);
12027 default: return Register();
12028 }
12029}
12030
12031// FastEmit functions for ISD::GET_ACTIVE_LANE_MASK.
12032
12033Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv2i1_rr(Register Op0, Register Op1) {
12034 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12035 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_D, RC: &AArch64::PPRRegClass, Op0, Op1);
12036 }
12037 return Register();
12038}
12039
12040Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv4i1_rr(Register Op0, Register Op1) {
12041 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12042 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_S, RC: &AArch64::PPRRegClass, Op0, Op1);
12043 }
12044 return Register();
12045}
12046
12047Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv8i1_rr(Register Op0, Register Op1) {
12048 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12049 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_H, RC: &AArch64::PPRRegClass, Op0, Op1);
12050 }
12051 return Register();
12052}
12053
12054Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv16i1_rr(Register Op0, Register Op1) {
12055 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12056 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_B, RC: &AArch64::PPRRegClass, Op0, Op1);
12057 }
12058 return Register();
12059}
12060
12061Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12062switch (RetVT.SimpleTy) {
12063 case MVT::nxv2i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv2i1_rr(Op0, Op1);
12064 case MVT::nxv4i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv4i1_rr(Op0, Op1);
12065 case MVT::nxv8i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv8i1_rr(Op0, Op1);
12066 case MVT::nxv16i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv16i1_rr(Op0, Op1);
12067 default: return Register();
12068}
12069}
12070
12071Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv2i1_rr(Register Op0, Register Op1) {
12072 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12073 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_D, RC: &AArch64::PPRRegClass, Op0, Op1);
12074 }
12075 return Register();
12076}
12077
12078Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv4i1_rr(Register Op0, Register Op1) {
12079 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12080 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_S, RC: &AArch64::PPRRegClass, Op0, Op1);
12081 }
12082 return Register();
12083}
12084
12085Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv8i1_rr(Register Op0, Register Op1) {
12086 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12087 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_H, RC: &AArch64::PPRRegClass, Op0, Op1);
12088 }
12089 return Register();
12090}
12091
12092Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv16i1_rr(Register Op0, Register Op1) {
12093 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12094 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_B, RC: &AArch64::PPRRegClass, Op0, Op1);
12095 }
12096 return Register();
12097}
12098
12099Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12100switch (RetVT.SimpleTy) {
12101 case MVT::nxv2i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv2i1_rr(Op0, Op1);
12102 case MVT::nxv4i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv4i1_rr(Op0, Op1);
12103 case MVT::nxv8i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv8i1_rr(Op0, Op1);
12104 case MVT::nxv16i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv16i1_rr(Op0, Op1);
12105 default: return Register();
12106}
12107}
12108
12109Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12110 switch (VT.SimpleTy) {
12111 case MVT::i32: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_rr(RetVT, Op0, Op1);
12112 case MVT::i64: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_rr(RetVT, Op0, Op1);
12113 default: return Register();
12114 }
12115}
12116
12117// FastEmit functions for ISD::MUL.
12118
12119Register fastEmit_ISD_MUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12120 if (RetVT.SimpleTy != MVT::v8i8)
12121 return Register();
12122 if ((Subtarget->isNeonAvailable())) {
12123 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12124 }
12125 return Register();
12126}
12127
12128Register fastEmit_ISD_MUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12129 if (RetVT.SimpleTy != MVT::v16i8)
12130 return Register();
12131 if ((Subtarget->isNeonAvailable())) {
12132 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12133 }
12134 return Register();
12135}
12136
12137Register fastEmit_ISD_MUL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12138 if (RetVT.SimpleTy != MVT::v4i16)
12139 return Register();
12140 if ((Subtarget->isNeonAvailable())) {
12141 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12142 }
12143 return Register();
12144}
12145
12146Register fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12147 if (RetVT.SimpleTy != MVT::v8i16)
12148 return Register();
12149 if ((Subtarget->isNeonAvailable())) {
12150 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12151 }
12152 return Register();
12153}
12154
12155Register fastEmit_ISD_MUL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12156 if (RetVT.SimpleTy != MVT::v2i32)
12157 return Register();
12158 if ((Subtarget->isNeonAvailable())) {
12159 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12160 }
12161 return Register();
12162}
12163
12164Register fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12165 if (RetVT.SimpleTy != MVT::v4i32)
12166 return Register();
12167 if ((Subtarget->isNeonAvailable())) {
12168 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12169 }
12170 return Register();
12171}
12172
12173Register fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12174 switch (VT.SimpleTy) {
12175 case MVT::v8i8: return fastEmit_ISD_MUL_MVT_v8i8_rr(RetVT, Op0, Op1);
12176 case MVT::v16i8: return fastEmit_ISD_MUL_MVT_v16i8_rr(RetVT, Op0, Op1);
12177 case MVT::v4i16: return fastEmit_ISD_MUL_MVT_v4i16_rr(RetVT, Op0, Op1);
12178 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
12179 case MVT::v2i32: return fastEmit_ISD_MUL_MVT_v2i32_rr(RetVT, Op0, Op1);
12180 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
12181 default: return Register();
12182 }
12183}
12184
12185// FastEmit functions for ISD::MULHS.
12186
12187Register fastEmit_ISD_MULHS_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12188 if (RetVT.SimpleTy != MVT::i64)
12189 return Register();
12190 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULHrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12191}
12192
12193Register fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12194 switch (VT.SimpleTy) {
12195 case MVT::i64: return fastEmit_ISD_MULHS_MVT_i64_rr(RetVT, Op0, Op1);
12196 default: return Register();
12197 }
12198}
12199
12200// FastEmit functions for ISD::MULHU.
12201
12202Register fastEmit_ISD_MULHU_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12203 if (RetVT.SimpleTy != MVT::i64)
12204 return Register();
12205 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULHrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12206}
12207
12208Register fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12209 switch (VT.SimpleTy) {
12210 case MVT::i64: return fastEmit_ISD_MULHU_MVT_i64_rr(RetVT, Op0, Op1);
12211 default: return Register();
12212 }
12213}
12214
12215// FastEmit functions for ISD::OR.
12216
12217Register fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12218 if (RetVT.SimpleTy != MVT::i32)
12219 return Register();
12220 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12221}
12222
12223Register fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12224 if (RetVT.SimpleTy != MVT::i64)
12225 return Register();
12226 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12227}
12228
12229Register fastEmit_ISD_OR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12230 if (RetVT.SimpleTy != MVT::v8i8)
12231 return Register();
12232 if ((Subtarget->isNeonAvailable())) {
12233 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12234 }
12235 return Register();
12236}
12237
12238Register fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12239 if (RetVT.SimpleTy != MVT::v16i8)
12240 return Register();
12241 if ((Subtarget->isNeonAvailable())) {
12242 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12243 }
12244 return Register();
12245}
12246
12247Register fastEmit_ISD_OR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12248 if (RetVT.SimpleTy != MVT::v4i16)
12249 return Register();
12250 if ((Subtarget->isNeonAvailable())) {
12251 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12252 }
12253 return Register();
12254}
12255
12256Register fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12257 if (RetVT.SimpleTy != MVT::v8i16)
12258 return Register();
12259 if ((Subtarget->isNeonAvailable())) {
12260 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12261 }
12262 return Register();
12263}
12264
12265Register fastEmit_ISD_OR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12266 if (RetVT.SimpleTy != MVT::v2i32)
12267 return Register();
12268 if ((Subtarget->isNeonAvailable())) {
12269 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12270 }
12271 return Register();
12272}
12273
12274Register fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12275 if (RetVT.SimpleTy != MVT::v4i32)
12276 return Register();
12277 if ((Subtarget->isNeonAvailable())) {
12278 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12279 }
12280 return Register();
12281}
12282
12283Register fastEmit_ISD_OR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12284 if (RetVT.SimpleTy != MVT::v1i64)
12285 return Register();
12286 if ((Subtarget->isNeonAvailable())) {
12287 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12288 }
12289 return Register();
12290}
12291
12292Register fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12293 if (RetVT.SimpleTy != MVT::v2i64)
12294 return Register();
12295 if ((Subtarget->isNeonAvailable())) {
12296 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12297 }
12298 return Register();
12299}
12300
12301Register fastEmit_ISD_OR_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12302 if (RetVT.SimpleTy != MVT::nxv16i8)
12303 return Register();
12304 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12305 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12306 }
12307 return Register();
12308}
12309
12310Register fastEmit_ISD_OR_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12311 if (RetVT.SimpleTy != MVT::nxv8i16)
12312 return Register();
12313 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12314 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12315 }
12316 return Register();
12317}
12318
12319Register fastEmit_ISD_OR_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12320 if (RetVT.SimpleTy != MVT::nxv4i32)
12321 return Register();
12322 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12323 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12324 }
12325 return Register();
12326}
12327
12328Register fastEmit_ISD_OR_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12329 if (RetVT.SimpleTy != MVT::nxv2i64)
12330 return Register();
12331 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12332 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12333 }
12334 return Register();
12335}
12336
12337Register fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12338 switch (VT.SimpleTy) {
12339 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
12340 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op1);
12341 case MVT::v8i8: return fastEmit_ISD_OR_MVT_v8i8_rr(RetVT, Op0, Op1);
12342 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
12343 case MVT::v4i16: return fastEmit_ISD_OR_MVT_v4i16_rr(RetVT, Op0, Op1);
12344 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
12345 case MVT::v2i32: return fastEmit_ISD_OR_MVT_v2i32_rr(RetVT, Op0, Op1);
12346 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
12347 case MVT::v1i64: return fastEmit_ISD_OR_MVT_v1i64_rr(RetVT, Op0, Op1);
12348 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
12349 case MVT::nxv16i8: return fastEmit_ISD_OR_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12350 case MVT::nxv8i16: return fastEmit_ISD_OR_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12351 case MVT::nxv4i32: return fastEmit_ISD_OR_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12352 case MVT::nxv2i64: return fastEmit_ISD_OR_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12353 default: return Register();
12354 }
12355}
12356
12357// FastEmit functions for ISD::ROTR.
12358
12359Register fastEmit_ISD_ROTR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12360 if (RetVT.SimpleTy != MVT::i64)
12361 return Register();
12362 return fastEmitInst_rr(MachineInstOpcode: AArch64::RORVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12363}
12364
12365Register fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12366 switch (VT.SimpleTy) {
12367 case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_rr(RetVT, Op0, Op1);
12368 default: return Register();
12369 }
12370}
12371
12372// FastEmit functions for ISD::SADDSAT.
12373
12374Register fastEmit_ISD_SADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12375 if (RetVT.SimpleTy != MVT::v8i8)
12376 return Register();
12377 if ((Subtarget->isNeonAvailable())) {
12378 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12379 }
12380 return Register();
12381}
12382
12383Register fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12384 if (RetVT.SimpleTy != MVT::v16i8)
12385 return Register();
12386 if ((Subtarget->isNeonAvailable())) {
12387 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12388 }
12389 return Register();
12390}
12391
12392Register fastEmit_ISD_SADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12393 if (RetVT.SimpleTy != MVT::v4i16)
12394 return Register();
12395 if ((Subtarget->isNeonAvailable())) {
12396 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12397 }
12398 return Register();
12399}
12400
12401Register fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12402 if (RetVT.SimpleTy != MVT::v8i16)
12403 return Register();
12404 if ((Subtarget->isNeonAvailable())) {
12405 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12406 }
12407 return Register();
12408}
12409
12410Register fastEmit_ISD_SADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12411 if (RetVT.SimpleTy != MVT::v2i32)
12412 return Register();
12413 if ((Subtarget->isNeonAvailable())) {
12414 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12415 }
12416 return Register();
12417}
12418
12419Register fastEmit_ISD_SADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12420 if (RetVT.SimpleTy != MVT::v4i32)
12421 return Register();
12422 if ((Subtarget->isNeonAvailable())) {
12423 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12424 }
12425 return Register();
12426}
12427
12428Register fastEmit_ISD_SADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12429 if (RetVT.SimpleTy != MVT::v1i64)
12430 return Register();
12431 if ((Subtarget->isNeonAvailable())) {
12432 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
12433 }
12434 return Register();
12435}
12436
12437Register fastEmit_ISD_SADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12438 if (RetVT.SimpleTy != MVT::v2i64)
12439 return Register();
12440 if ((Subtarget->isNeonAvailable())) {
12441 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12442 }
12443 return Register();
12444}
12445
12446Register fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12447 if (RetVT.SimpleTy != MVT::nxv16i8)
12448 return Register();
12449 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12450 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
12451 }
12452 return Register();
12453}
12454
12455Register fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12456 if (RetVT.SimpleTy != MVT::nxv8i16)
12457 return Register();
12458 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12459 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
12460 }
12461 return Register();
12462}
12463
12464Register fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12465 if (RetVT.SimpleTy != MVT::nxv4i32)
12466 return Register();
12467 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12468 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
12469 }
12470 return Register();
12471}
12472
12473Register fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12474 if (RetVT.SimpleTy != MVT::nxv2i64)
12475 return Register();
12476 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12477 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
12478 }
12479 return Register();
12480}
12481
12482Register fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12483 switch (VT.SimpleTy) {
12484 case MVT::v8i8: return fastEmit_ISD_SADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
12485 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12486 case MVT::v4i16: return fastEmit_ISD_SADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
12487 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12488 case MVT::v2i32: return fastEmit_ISD_SADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
12489 case MVT::v4i32: return fastEmit_ISD_SADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
12490 case MVT::v1i64: return fastEmit_ISD_SADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
12491 case MVT::v2i64: return fastEmit_ISD_SADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
12492 case MVT::nxv16i8: return fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12493 case MVT::nxv8i16: return fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12494 case MVT::nxv4i32: return fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12495 case MVT::nxv2i64: return fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12496 default: return Register();
12497 }
12498}
12499
12500// FastEmit functions for ISD::SDIV.
12501
12502Register fastEmit_ISD_SDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12503 if (RetVT.SimpleTy != MVT::i32)
12504 return Register();
12505 return fastEmitInst_rr(MachineInstOpcode: AArch64::SDIVWr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12506}
12507
12508Register fastEmit_ISD_SDIV_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12509 if (RetVT.SimpleTy != MVT::i64)
12510 return Register();
12511 return fastEmitInst_rr(MachineInstOpcode: AArch64::SDIVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12512}
12513
12514Register fastEmit_ISD_SDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12515 switch (VT.SimpleTy) {
12516 case MVT::i32: return fastEmit_ISD_SDIV_MVT_i32_rr(RetVT, Op0, Op1);
12517 case MVT::i64: return fastEmit_ISD_SDIV_MVT_i64_rr(RetVT, Op0, Op1);
12518 default: return Register();
12519 }
12520}
12521
12522// FastEmit functions for ISD::SHL.
12523
12524Register fastEmit_ISD_SHL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12525 if (RetVT.SimpleTy != MVT::i64)
12526 return Register();
12527 return fastEmitInst_rr(MachineInstOpcode: AArch64::LSLVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12528}
12529
12530Register fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12531 switch (VT.SimpleTy) {
12532 case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_rr(RetVT, Op0, Op1);
12533 default: return Register();
12534 }
12535}
12536
12537// FastEmit functions for ISD::SMAX.
12538
12539Register fastEmit_ISD_SMAX_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12540 if (RetVT.SimpleTy != MVT::i32)
12541 return Register();
12542 if ((Subtarget->hasCSSC())) {
12543 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12544 }
12545 return Register();
12546}
12547
12548Register fastEmit_ISD_SMAX_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12549 if (RetVT.SimpleTy != MVT::i64)
12550 return Register();
12551 if ((Subtarget->hasCSSC())) {
12552 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12553 }
12554 return Register();
12555}
12556
12557Register fastEmit_ISD_SMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12558 if (RetVT.SimpleTy != MVT::v8i8)
12559 return Register();
12560 if ((Subtarget->isNeonAvailable())) {
12561 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12562 }
12563 return Register();
12564}
12565
12566Register fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12567 if (RetVT.SimpleTy != MVT::v16i8)
12568 return Register();
12569 if ((Subtarget->isNeonAvailable())) {
12570 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12571 }
12572 return Register();
12573}
12574
12575Register fastEmit_ISD_SMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12576 if (RetVT.SimpleTy != MVT::v4i16)
12577 return Register();
12578 if ((Subtarget->isNeonAvailable())) {
12579 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12580 }
12581 return Register();
12582}
12583
12584Register fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12585 if (RetVT.SimpleTy != MVT::v8i16)
12586 return Register();
12587 if ((Subtarget->isNeonAvailable())) {
12588 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12589 }
12590 return Register();
12591}
12592
12593Register fastEmit_ISD_SMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12594 if (RetVT.SimpleTy != MVT::v2i32)
12595 return Register();
12596 if ((Subtarget->isNeonAvailable())) {
12597 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12598 }
12599 return Register();
12600}
12601
12602Register fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12603 if (RetVT.SimpleTy != MVT::v4i32)
12604 return Register();
12605 if ((Subtarget->isNeonAvailable())) {
12606 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12607 }
12608 return Register();
12609}
12610
12611Register fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12612 switch (VT.SimpleTy) {
12613 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_rr(RetVT, Op0, Op1);
12614 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_rr(RetVT, Op0, Op1);
12615 case MVT::v8i8: return fastEmit_ISD_SMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
12616 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
12617 case MVT::v4i16: return fastEmit_ISD_SMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
12618 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
12619 case MVT::v2i32: return fastEmit_ISD_SMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
12620 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
12621 default: return Register();
12622 }
12623}
12624
12625// FastEmit functions for ISD::SMIN.
12626
12627Register fastEmit_ISD_SMIN_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12628 if (RetVT.SimpleTy != MVT::i32)
12629 return Register();
12630 if ((Subtarget->hasCSSC())) {
12631 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12632 }
12633 return Register();
12634}
12635
12636Register fastEmit_ISD_SMIN_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12637 if (RetVT.SimpleTy != MVT::i64)
12638 return Register();
12639 if ((Subtarget->hasCSSC())) {
12640 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12641 }
12642 return Register();
12643}
12644
12645Register fastEmit_ISD_SMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12646 if (RetVT.SimpleTy != MVT::v8i8)
12647 return Register();
12648 if ((Subtarget->isNeonAvailable())) {
12649 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12650 }
12651 return Register();
12652}
12653
12654Register fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12655 if (RetVT.SimpleTy != MVT::v16i8)
12656 return Register();
12657 if ((Subtarget->isNeonAvailable())) {
12658 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12659 }
12660 return Register();
12661}
12662
12663Register fastEmit_ISD_SMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12664 if (RetVT.SimpleTy != MVT::v4i16)
12665 return Register();
12666 if ((Subtarget->isNeonAvailable())) {
12667 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12668 }
12669 return Register();
12670}
12671
12672Register fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12673 if (RetVT.SimpleTy != MVT::v8i16)
12674 return Register();
12675 if ((Subtarget->isNeonAvailable())) {
12676 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12677 }
12678 return Register();
12679}
12680
12681Register fastEmit_ISD_SMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12682 if (RetVT.SimpleTy != MVT::v2i32)
12683 return Register();
12684 if ((Subtarget->isNeonAvailable())) {
12685 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12686 }
12687 return Register();
12688}
12689
12690Register fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12691 if (RetVT.SimpleTy != MVT::v4i32)
12692 return Register();
12693 if ((Subtarget->isNeonAvailable())) {
12694 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12695 }
12696 return Register();
12697}
12698
12699Register fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12700 switch (VT.SimpleTy) {
12701 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_rr(RetVT, Op0, Op1);
12702 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_rr(RetVT, Op0, Op1);
12703 case MVT::v8i8: return fastEmit_ISD_SMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
12704 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
12705 case MVT::v4i16: return fastEmit_ISD_SMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
12706 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
12707 case MVT::v2i32: return fastEmit_ISD_SMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
12708 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
12709 default: return Register();
12710 }
12711}
12712
12713// FastEmit functions for ISD::SRA.
12714
12715Register fastEmit_ISD_SRA_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12716 if (RetVT.SimpleTy != MVT::i64)
12717 return Register();
12718 return fastEmitInst_rr(MachineInstOpcode: AArch64::ASRVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12719}
12720
12721Register fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12722 switch (VT.SimpleTy) {
12723 case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_rr(RetVT, Op0, Op1);
12724 default: return Register();
12725 }
12726}
12727
12728// FastEmit functions for ISD::SRL.
12729
12730Register fastEmit_ISD_SRL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12731 if (RetVT.SimpleTy != MVT::i64)
12732 return Register();
12733 return fastEmitInst_rr(MachineInstOpcode: AArch64::LSRVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12734}
12735
12736Register fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12737 switch (VT.SimpleTy) {
12738 case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_rr(RetVT, Op0, Op1);
12739 default: return Register();
12740 }
12741}
12742
12743// FastEmit functions for ISD::SSUBSAT.
12744
12745Register fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12746 if (RetVT.SimpleTy != MVT::v8i8)
12747 return Register();
12748 if ((Subtarget->isNeonAvailable())) {
12749 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12750 }
12751 return Register();
12752}
12753
12754Register fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12755 if (RetVT.SimpleTy != MVT::v16i8)
12756 return Register();
12757 if ((Subtarget->isNeonAvailable())) {
12758 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12759 }
12760 return Register();
12761}
12762
12763Register fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12764 if (RetVT.SimpleTy != MVT::v4i16)
12765 return Register();
12766 if ((Subtarget->isNeonAvailable())) {
12767 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12768 }
12769 return Register();
12770}
12771
12772Register fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12773 if (RetVT.SimpleTy != MVT::v8i16)
12774 return Register();
12775 if ((Subtarget->isNeonAvailable())) {
12776 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12777 }
12778 return Register();
12779}
12780
12781Register fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12782 if (RetVT.SimpleTy != MVT::v2i32)
12783 return Register();
12784 if ((Subtarget->isNeonAvailable())) {
12785 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12786 }
12787 return Register();
12788}
12789
12790Register fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12791 if (RetVT.SimpleTy != MVT::v4i32)
12792 return Register();
12793 if ((Subtarget->isNeonAvailable())) {
12794 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12795 }
12796 return Register();
12797}
12798
12799Register fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12800 if (RetVT.SimpleTy != MVT::v1i64)
12801 return Register();
12802 if ((Subtarget->isNeonAvailable())) {
12803 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
12804 }
12805 return Register();
12806}
12807
12808Register fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12809 if (RetVT.SimpleTy != MVT::v2i64)
12810 return Register();
12811 if ((Subtarget->isNeonAvailable())) {
12812 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12813 }
12814 return Register();
12815}
12816
12817Register fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12818 if (RetVT.SimpleTy != MVT::nxv16i8)
12819 return Register();
12820 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12821 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
12822 }
12823 return Register();
12824}
12825
12826Register fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12827 if (RetVT.SimpleTy != MVT::nxv8i16)
12828 return Register();
12829 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12830 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
12831 }
12832 return Register();
12833}
12834
12835Register fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12836 if (RetVT.SimpleTy != MVT::nxv4i32)
12837 return Register();
12838 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12839 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
12840 }
12841 return Register();
12842}
12843
12844Register fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12845 if (RetVT.SimpleTy != MVT::nxv2i64)
12846 return Register();
12847 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12848 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
12849 }
12850 return Register();
12851}
12852
12853Register fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12854 switch (VT.SimpleTy) {
12855 case MVT::v8i8: return fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
12856 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12857 case MVT::v4i16: return fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
12858 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12859 case MVT::v2i32: return fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
12860 case MVT::v4i32: return fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
12861 case MVT::v1i64: return fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
12862 case MVT::v2i64: return fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
12863 case MVT::nxv16i8: return fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12864 case MVT::nxv8i16: return fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12865 case MVT::nxv4i32: return fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12866 case MVT::nxv2i64: return fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12867 default: return Register();
12868 }
12869}
12870
12871// FastEmit functions for ISD::STRICT_FADD.
12872
12873Register fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12874 if (RetVT.SimpleTy != MVT::f16)
12875 return Register();
12876 if ((Subtarget->hasFullFP16())) {
12877 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12878 }
12879 return Register();
12880}
12881
12882Register fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12883 if (RetVT.SimpleTy != MVT::f32)
12884 return Register();
12885 if ((Subtarget->hasFPARMv8())) {
12886 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12887 }
12888 return Register();
12889}
12890
12891Register fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12892 if (RetVT.SimpleTy != MVT::f64)
12893 return Register();
12894 if ((Subtarget->hasFPARMv8())) {
12895 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12896 }
12897 return Register();
12898}
12899
12900Register fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12901 if (RetVT.SimpleTy != MVT::v4f16)
12902 return Register();
12903 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12904 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12905 }
12906 return Register();
12907}
12908
12909Register fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12910 if (RetVT.SimpleTy != MVT::v8f16)
12911 return Register();
12912 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12913 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12914 }
12915 return Register();
12916}
12917
12918Register fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12919 if (RetVT.SimpleTy != MVT::v2f32)
12920 return Register();
12921 if ((Subtarget->isNeonAvailable())) {
12922 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12923 }
12924 return Register();
12925}
12926
12927Register fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12928 if (RetVT.SimpleTy != MVT::v4f32)
12929 return Register();
12930 if ((Subtarget->isNeonAvailable())) {
12931 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12932 }
12933 return Register();
12934}
12935
12936Register fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12937 if (RetVT.SimpleTy != MVT::v2f64)
12938 return Register();
12939 if ((Subtarget->isNeonAvailable())) {
12940 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12941 }
12942 return Register();
12943}
12944
12945Register fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12946 switch (VT.SimpleTy) {
12947 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1);
12948 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1);
12949 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1);
12950 case MVT::v4f16: return fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
12951 case MVT::v8f16: return fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
12952 case MVT::v2f32: return fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
12953 case MVT::v4f32: return fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
12954 case MVT::v2f64: return fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
12955 default: return Register();
12956 }
12957}
12958
12959// FastEmit functions for ISD::STRICT_FDIV.
12960
12961Register fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12962 if (RetVT.SimpleTy != MVT::f16)
12963 return Register();
12964 if ((Subtarget->hasFullFP16())) {
12965 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12966 }
12967 return Register();
12968}
12969
12970Register fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12971 if (RetVT.SimpleTy != MVT::f32)
12972 return Register();
12973 if ((Subtarget->hasFPARMv8())) {
12974 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12975 }
12976 return Register();
12977}
12978
12979Register fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12980 if (RetVT.SimpleTy != MVT::f64)
12981 return Register();
12982 if ((Subtarget->hasFPARMv8())) {
12983 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12984 }
12985 return Register();
12986}
12987
12988Register fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12989 if (RetVT.SimpleTy != MVT::v4f16)
12990 return Register();
12991 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12992 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12993 }
12994 return Register();
12995}
12996
12997Register fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12998 if (RetVT.SimpleTy != MVT::v8f16)
12999 return Register();
13000 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13001 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13002 }
13003 return Register();
13004}
13005
13006Register fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13007 if (RetVT.SimpleTy != MVT::v2f32)
13008 return Register();
13009 if ((Subtarget->isNeonAvailable())) {
13010 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13011 }
13012 return Register();
13013}
13014
13015Register fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13016 if (RetVT.SimpleTy != MVT::v4f32)
13017 return Register();
13018 if ((Subtarget->isNeonAvailable())) {
13019 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13020 }
13021 return Register();
13022}
13023
13024Register fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13025 if (RetVT.SimpleTy != MVT::v2f64)
13026 return Register();
13027 if ((Subtarget->isNeonAvailable())) {
13028 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13029 }
13030 return Register();
13031}
13032
13033Register fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13034 switch (VT.SimpleTy) {
13035 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
13036 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
13037 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
13038 case MVT::v4f16: return fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1);
13039 case MVT::v8f16: return fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
13040 case MVT::v2f32: return fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1);
13041 case MVT::v4f32: return fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
13042 case MVT::v2f64: return fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
13043 default: return Register();
13044 }
13045}
13046
13047// FastEmit functions for ISD::STRICT_FMAXIMUM.
13048
13049Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13050 if (RetVT.SimpleTy != MVT::f16)
13051 return Register();
13052 if ((Subtarget->hasFullFP16())) {
13053 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13054 }
13055 return Register();
13056}
13057
13058Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13059 if (RetVT.SimpleTy != MVT::f32)
13060 return Register();
13061 if ((Subtarget->hasFPARMv8())) {
13062 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13063 }
13064 return Register();
13065}
13066
13067Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13068 if (RetVT.SimpleTy != MVT::f64)
13069 return Register();
13070 if ((Subtarget->hasFPARMv8())) {
13071 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13072 }
13073 return Register();
13074}
13075
13076Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13077 if (RetVT.SimpleTy != MVT::v4f16)
13078 return Register();
13079 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13080 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13081 }
13082 return Register();
13083}
13084
13085Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13086 if (RetVT.SimpleTy != MVT::v8f16)
13087 return Register();
13088 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13089 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13090 }
13091 return Register();
13092}
13093
13094Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13095 if (RetVT.SimpleTy != MVT::v2f32)
13096 return Register();
13097 if ((Subtarget->isNeonAvailable())) {
13098 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13099 }
13100 return Register();
13101}
13102
13103Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13104 if (RetVT.SimpleTy != MVT::v4f32)
13105 return Register();
13106 if ((Subtarget->isNeonAvailable())) {
13107 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13108 }
13109 return Register();
13110}
13111
13112Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13113 if (RetVT.SimpleTy != MVT::v2f64)
13114 return Register();
13115 if ((Subtarget->isNeonAvailable())) {
13116 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13117 }
13118 return Register();
13119}
13120
13121Register fastEmit_ISD_STRICT_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13122 switch (VT.SimpleTy) {
13123 case MVT::f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1);
13124 case MVT::f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1);
13125 case MVT::f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1);
13126 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13127 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13128 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13129 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13130 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13131 default: return Register();
13132 }
13133}
13134
13135// FastEmit functions for ISD::STRICT_FMAXNUM.
13136
13137Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13138 if (RetVT.SimpleTy != MVT::f16)
13139 return Register();
13140 if ((Subtarget->hasFullFP16())) {
13141 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13142 }
13143 return Register();
13144}
13145
13146Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13147 if (RetVT.SimpleTy != MVT::f32)
13148 return Register();
13149 if ((Subtarget->hasFPARMv8())) {
13150 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13151 }
13152 return Register();
13153}
13154
13155Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13156 if (RetVT.SimpleTy != MVT::f64)
13157 return Register();
13158 if ((Subtarget->hasFPARMv8())) {
13159 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13160 }
13161 return Register();
13162}
13163
13164Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13165 if (RetVT.SimpleTy != MVT::v4f16)
13166 return Register();
13167 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13168 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13169 }
13170 return Register();
13171}
13172
13173Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13174 if (RetVT.SimpleTy != MVT::v8f16)
13175 return Register();
13176 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13177 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13178 }
13179 return Register();
13180}
13181
13182Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13183 if (RetVT.SimpleTy != MVT::v2f32)
13184 return Register();
13185 if ((Subtarget->isNeonAvailable())) {
13186 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13187 }
13188 return Register();
13189}
13190
13191Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13192 if (RetVT.SimpleTy != MVT::v4f32)
13193 return Register();
13194 if ((Subtarget->isNeonAvailable())) {
13195 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13196 }
13197 return Register();
13198}
13199
13200Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13201 if (RetVT.SimpleTy != MVT::v2f64)
13202 return Register();
13203 if ((Subtarget->isNeonAvailable())) {
13204 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13205 }
13206 return Register();
13207}
13208
13209Register fastEmit_ISD_STRICT_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13210 switch (VT.SimpleTy) {
13211 case MVT::f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
13212 case MVT::f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
13213 case MVT::f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
13214 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13215 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13216 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13217 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13218 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13219 default: return Register();
13220 }
13221}
13222
13223// FastEmit functions for ISD::STRICT_FMINIMUM.
13224
13225Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13226 if (RetVT.SimpleTy != MVT::f16)
13227 return Register();
13228 if ((Subtarget->hasFullFP16())) {
13229 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13230 }
13231 return Register();
13232}
13233
13234Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13235 if (RetVT.SimpleTy != MVT::f32)
13236 return Register();
13237 if ((Subtarget->hasFPARMv8())) {
13238 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13239 }
13240 return Register();
13241}
13242
13243Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13244 if (RetVT.SimpleTy != MVT::f64)
13245 return Register();
13246 if ((Subtarget->hasFPARMv8())) {
13247 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13248 }
13249 return Register();
13250}
13251
13252Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13253 if (RetVT.SimpleTy != MVT::v4f16)
13254 return Register();
13255 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13256 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13257 }
13258 return Register();
13259}
13260
13261Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13262 if (RetVT.SimpleTy != MVT::v8f16)
13263 return Register();
13264 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13265 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13266 }
13267 return Register();
13268}
13269
13270Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13271 if (RetVT.SimpleTy != MVT::v2f32)
13272 return Register();
13273 if ((Subtarget->isNeonAvailable())) {
13274 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13275 }
13276 return Register();
13277}
13278
13279Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13280 if (RetVT.SimpleTy != MVT::v4f32)
13281 return Register();
13282 if ((Subtarget->isNeonAvailable())) {
13283 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13284 }
13285 return Register();
13286}
13287
13288Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13289 if (RetVT.SimpleTy != MVT::v2f64)
13290 return Register();
13291 if ((Subtarget->isNeonAvailable())) {
13292 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13293 }
13294 return Register();
13295}
13296
13297Register fastEmit_ISD_STRICT_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13298 switch (VT.SimpleTy) {
13299 case MVT::f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1);
13300 case MVT::f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1);
13301 case MVT::f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1);
13302 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13303 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13304 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13305 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13306 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13307 default: return Register();
13308 }
13309}
13310
13311// FastEmit functions for ISD::STRICT_FMINNUM.
13312
13313Register fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13314 if (RetVT.SimpleTy != MVT::f16)
13315 return Register();
13316 if ((Subtarget->hasFullFP16())) {
13317 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13318 }
13319 return Register();
13320}
13321
13322Register fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13323 if (RetVT.SimpleTy != MVT::f32)
13324 return Register();
13325 if ((Subtarget->hasFPARMv8())) {
13326 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13327 }
13328 return Register();
13329}
13330
13331Register fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13332 if (RetVT.SimpleTy != MVT::f64)
13333 return Register();
13334 if ((Subtarget->hasFPARMv8())) {
13335 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13336 }
13337 return Register();
13338}
13339
13340Register fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13341 if (RetVT.SimpleTy != MVT::v4f16)
13342 return Register();
13343 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13344 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13345 }
13346 return Register();
13347}
13348
13349Register fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13350 if (RetVT.SimpleTy != MVT::v8f16)
13351 return Register();
13352 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13353 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13354 }
13355 return Register();
13356}
13357
13358Register fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13359 if (RetVT.SimpleTy != MVT::v2f32)
13360 return Register();
13361 if ((Subtarget->isNeonAvailable())) {
13362 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13363 }
13364 return Register();
13365}
13366
13367Register fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13368 if (RetVT.SimpleTy != MVT::v4f32)
13369 return Register();
13370 if ((Subtarget->isNeonAvailable())) {
13371 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13372 }
13373 return Register();
13374}
13375
13376Register fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13377 if (RetVT.SimpleTy != MVT::v2f64)
13378 return Register();
13379 if ((Subtarget->isNeonAvailable())) {
13380 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13381 }
13382 return Register();
13383}
13384
13385Register fastEmit_ISD_STRICT_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13386 switch (VT.SimpleTy) {
13387 case MVT::f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
13388 case MVT::f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
13389 case MVT::f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
13390 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13391 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13392 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13393 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13394 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13395 default: return Register();
13396 }
13397}
13398
13399// FastEmit functions for ISD::STRICT_FMUL.
13400
13401Register fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13402 if (RetVT.SimpleTy != MVT::f16)
13403 return Register();
13404 if ((Subtarget->hasFullFP16())) {
13405 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13406 }
13407 return Register();
13408}
13409
13410Register fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13411 if (RetVT.SimpleTy != MVT::f32)
13412 return Register();
13413 if ((Subtarget->hasFPARMv8())) {
13414 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13415 }
13416 return Register();
13417}
13418
13419Register fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13420 if (RetVT.SimpleTy != MVT::f64)
13421 return Register();
13422 if ((Subtarget->hasFPARMv8())) {
13423 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13424 }
13425 return Register();
13426}
13427
13428Register fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13429 if (RetVT.SimpleTy != MVT::v4f16)
13430 return Register();
13431 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13432 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13433 }
13434 return Register();
13435}
13436
13437Register fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13438 if (RetVT.SimpleTy != MVT::v8f16)
13439 return Register();
13440 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13441 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13442 }
13443 return Register();
13444}
13445
13446Register fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13447 if (RetVT.SimpleTy != MVT::v2f32)
13448 return Register();
13449 if ((Subtarget->isNeonAvailable())) {
13450 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13451 }
13452 return Register();
13453}
13454
13455Register fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13456 if (RetVT.SimpleTy != MVT::v4f32)
13457 return Register();
13458 if ((Subtarget->isNeonAvailable())) {
13459 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13460 }
13461 return Register();
13462}
13463
13464Register fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13465 if (RetVT.SimpleTy != MVT::v2f64)
13466 return Register();
13467 if ((Subtarget->isNeonAvailable())) {
13468 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13469 }
13470 return Register();
13471}
13472
13473Register fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13474 switch (VT.SimpleTy) {
13475 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
13476 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
13477 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
13478 case MVT::v4f16: return fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
13479 case MVT::v8f16: return fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
13480 case MVT::v2f32: return fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
13481 case MVT::v4f32: return fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
13482 case MVT::v2f64: return fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
13483 default: return Register();
13484 }
13485}
13486
13487// FastEmit functions for ISD::STRICT_FSUB.
13488
13489Register fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13490 if (RetVT.SimpleTy != MVT::f16)
13491 return Register();
13492 if ((Subtarget->hasFullFP16())) {
13493 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13494 }
13495 return Register();
13496}
13497
13498Register fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13499 if (RetVT.SimpleTy != MVT::f32)
13500 return Register();
13501 if ((Subtarget->hasFPARMv8())) {
13502 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13503 }
13504 return Register();
13505}
13506
13507Register fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13508 if (RetVT.SimpleTy != MVT::f64)
13509 return Register();
13510 if ((Subtarget->hasFPARMv8())) {
13511 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13512 }
13513 return Register();
13514}
13515
13516Register fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13517 if (RetVT.SimpleTy != MVT::v4f16)
13518 return Register();
13519 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13520 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13521 }
13522 return Register();
13523}
13524
13525Register fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13526 if (RetVT.SimpleTy != MVT::v8f16)
13527 return Register();
13528 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13529 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13530 }
13531 return Register();
13532}
13533
13534Register fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13535 if (RetVT.SimpleTy != MVT::v2f32)
13536 return Register();
13537 if ((Subtarget->isNeonAvailable())) {
13538 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13539 }
13540 return Register();
13541}
13542
13543Register fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13544 if (RetVT.SimpleTy != MVT::v4f32)
13545 return Register();
13546 if ((Subtarget->isNeonAvailable())) {
13547 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13548 }
13549 return Register();
13550}
13551
13552Register fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13553 if (RetVT.SimpleTy != MVT::v2f64)
13554 return Register();
13555 if ((Subtarget->isNeonAvailable())) {
13556 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13557 }
13558 return Register();
13559}
13560
13561Register fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13562 switch (VT.SimpleTy) {
13563 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
13564 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
13565 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
13566 case MVT::v4f16: return fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
13567 case MVT::v8f16: return fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
13568 case MVT::v2f32: return fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
13569 case MVT::v4f32: return fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
13570 case MVT::v2f64: return fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
13571 default: return Register();
13572 }
13573}
13574
13575// FastEmit functions for ISD::SUB.
13576
13577Register fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13578 if (RetVT.SimpleTy != MVT::i32)
13579 return Register();
13580 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBSWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13581}
13582
13583Register fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13584 if (RetVT.SimpleTy != MVT::i64)
13585 return Register();
13586 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBSXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13587}
13588
13589Register fastEmit_ISD_SUB_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13590 if (RetVT.SimpleTy != MVT::v8i8)
13591 return Register();
13592 if ((Subtarget->isNeonAvailable())) {
13593 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13594 }
13595 return Register();
13596}
13597
13598Register fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13599 if (RetVT.SimpleTy != MVT::v16i8)
13600 return Register();
13601 if ((Subtarget->isNeonAvailable())) {
13602 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13603 }
13604 return Register();
13605}
13606
13607Register fastEmit_ISD_SUB_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13608 if (RetVT.SimpleTy != MVT::v4i16)
13609 return Register();
13610 if ((Subtarget->isNeonAvailable())) {
13611 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13612 }
13613 return Register();
13614}
13615
13616Register fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13617 if (RetVT.SimpleTy != MVT::v8i16)
13618 return Register();
13619 if ((Subtarget->isNeonAvailable())) {
13620 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13621 }
13622 return Register();
13623}
13624
13625Register fastEmit_ISD_SUB_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13626 if (RetVT.SimpleTy != MVT::v2i32)
13627 return Register();
13628 if ((Subtarget->isNeonAvailable())) {
13629 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13630 }
13631 return Register();
13632}
13633
13634Register fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13635 if (RetVT.SimpleTy != MVT::v4i32)
13636 return Register();
13637 if ((Subtarget->isNeonAvailable())) {
13638 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13639 }
13640 return Register();
13641}
13642
13643Register fastEmit_ISD_SUB_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13644 if (RetVT.SimpleTy != MVT::v1i64)
13645 return Register();
13646 if ((Subtarget->isNeonAvailable())) {
13647 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13648 }
13649 return Register();
13650}
13651
13652Register fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13653 if (RetVT.SimpleTy != MVT::v2i64)
13654 return Register();
13655 if ((Subtarget->isNeonAvailable())) {
13656 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13657 }
13658 return Register();
13659}
13660
13661Register fastEmit_ISD_SUB_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13662 if (RetVT.SimpleTy != MVT::nxv16i8)
13663 return Register();
13664 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13665 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13666 }
13667 return Register();
13668}
13669
13670Register fastEmit_ISD_SUB_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13671 if (RetVT.SimpleTy != MVT::nxv8i16)
13672 return Register();
13673 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13674 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13675 }
13676 return Register();
13677}
13678
13679Register fastEmit_ISD_SUB_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13680 if (RetVT.SimpleTy != MVT::nxv4i32)
13681 return Register();
13682 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13683 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13684 }
13685 return Register();
13686}
13687
13688Register fastEmit_ISD_SUB_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13689 if (RetVT.SimpleTy != MVT::nxv2i64)
13690 return Register();
13691 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13692 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13693 }
13694 return Register();
13695}
13696
13697Register fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13698 switch (VT.SimpleTy) {
13699 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
13700 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op1);
13701 case MVT::v8i8: return fastEmit_ISD_SUB_MVT_v8i8_rr(RetVT, Op0, Op1);
13702 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
13703 case MVT::v4i16: return fastEmit_ISD_SUB_MVT_v4i16_rr(RetVT, Op0, Op1);
13704 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
13705 case MVT::v2i32: return fastEmit_ISD_SUB_MVT_v2i32_rr(RetVT, Op0, Op1);
13706 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
13707 case MVT::v1i64: return fastEmit_ISD_SUB_MVT_v1i64_rr(RetVT, Op0, Op1);
13708 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
13709 case MVT::nxv16i8: return fastEmit_ISD_SUB_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13710 case MVT::nxv8i16: return fastEmit_ISD_SUB_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13711 case MVT::nxv4i32: return fastEmit_ISD_SUB_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13712 case MVT::nxv2i64: return fastEmit_ISD_SUB_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13713 default: return Register();
13714 }
13715}
13716
13717// FastEmit functions for ISD::UADDSAT.
13718
13719Register fastEmit_ISD_UADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13720 if (RetVT.SimpleTy != MVT::v8i8)
13721 return Register();
13722 if ((Subtarget->isNeonAvailable())) {
13723 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13724 }
13725 return Register();
13726}
13727
13728Register fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13729 if (RetVT.SimpleTy != MVT::v16i8)
13730 return Register();
13731 if ((Subtarget->isNeonAvailable())) {
13732 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13733 }
13734 return Register();
13735}
13736
13737Register fastEmit_ISD_UADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13738 if (RetVT.SimpleTy != MVT::v4i16)
13739 return Register();
13740 if ((Subtarget->isNeonAvailable())) {
13741 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13742 }
13743 return Register();
13744}
13745
13746Register fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13747 if (RetVT.SimpleTy != MVT::v8i16)
13748 return Register();
13749 if ((Subtarget->isNeonAvailable())) {
13750 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13751 }
13752 return Register();
13753}
13754
13755Register fastEmit_ISD_UADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13756 if (RetVT.SimpleTy != MVT::v2i32)
13757 return Register();
13758 if ((Subtarget->isNeonAvailable())) {
13759 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13760 }
13761 return Register();
13762}
13763
13764Register fastEmit_ISD_UADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13765 if (RetVT.SimpleTy != MVT::v4i32)
13766 return Register();
13767 if ((Subtarget->isNeonAvailable())) {
13768 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13769 }
13770 return Register();
13771}
13772
13773Register fastEmit_ISD_UADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13774 if (RetVT.SimpleTy != MVT::v1i64)
13775 return Register();
13776 if ((Subtarget->isNeonAvailable())) {
13777 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13778 }
13779 return Register();
13780}
13781
13782Register fastEmit_ISD_UADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13783 if (RetVT.SimpleTy != MVT::v2i64)
13784 return Register();
13785 if ((Subtarget->isNeonAvailable())) {
13786 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13787 }
13788 return Register();
13789}
13790
13791Register fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13792 if (RetVT.SimpleTy != MVT::nxv16i8)
13793 return Register();
13794 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13795 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13796 }
13797 return Register();
13798}
13799
13800Register fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13801 if (RetVT.SimpleTy != MVT::nxv8i16)
13802 return Register();
13803 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13804 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13805 }
13806 return Register();
13807}
13808
13809Register fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13810 if (RetVT.SimpleTy != MVT::nxv4i32)
13811 return Register();
13812 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13813 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13814 }
13815 return Register();
13816}
13817
13818Register fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13819 if (RetVT.SimpleTy != MVT::nxv2i64)
13820 return Register();
13821 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13822 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13823 }
13824 return Register();
13825}
13826
13827Register fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13828 switch (VT.SimpleTy) {
13829 case MVT::v8i8: return fastEmit_ISD_UADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
13830 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13831 case MVT::v4i16: return fastEmit_ISD_UADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
13832 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13833 case MVT::v2i32: return fastEmit_ISD_UADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
13834 case MVT::v4i32: return fastEmit_ISD_UADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
13835 case MVT::v1i64: return fastEmit_ISD_UADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
13836 case MVT::v2i64: return fastEmit_ISD_UADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
13837 case MVT::nxv16i8: return fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13838 case MVT::nxv8i16: return fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13839 case MVT::nxv4i32: return fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13840 case MVT::nxv2i64: return fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13841 default: return Register();
13842 }
13843}
13844
13845// FastEmit functions for ISD::UDIV.
13846
13847Register fastEmit_ISD_UDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13848 if (RetVT.SimpleTy != MVT::i32)
13849 return Register();
13850 return fastEmitInst_rr(MachineInstOpcode: AArch64::UDIVWr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13851}
13852
13853Register fastEmit_ISD_UDIV_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13854 if (RetVT.SimpleTy != MVT::i64)
13855 return Register();
13856 return fastEmitInst_rr(MachineInstOpcode: AArch64::UDIVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13857}
13858
13859Register fastEmit_ISD_UDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13860 switch (VT.SimpleTy) {
13861 case MVT::i32: return fastEmit_ISD_UDIV_MVT_i32_rr(RetVT, Op0, Op1);
13862 case MVT::i64: return fastEmit_ISD_UDIV_MVT_i64_rr(RetVT, Op0, Op1);
13863 default: return Register();
13864 }
13865}
13866
13867// FastEmit functions for ISD::UMAX.
13868
13869Register fastEmit_ISD_UMAX_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13870 if (RetVT.SimpleTy != MVT::i32)
13871 return Register();
13872 if ((Subtarget->hasCSSC())) {
13873 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13874 }
13875 return Register();
13876}
13877
13878Register fastEmit_ISD_UMAX_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13879 if (RetVT.SimpleTy != MVT::i64)
13880 return Register();
13881 if ((Subtarget->hasCSSC())) {
13882 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13883 }
13884 return Register();
13885}
13886
13887Register fastEmit_ISD_UMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13888 if (RetVT.SimpleTy != MVT::v8i8)
13889 return Register();
13890 if ((Subtarget->isNeonAvailable())) {
13891 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13892 }
13893 return Register();
13894}
13895
13896Register fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13897 if (RetVT.SimpleTy != MVT::v16i8)
13898 return Register();
13899 if ((Subtarget->isNeonAvailable())) {
13900 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13901 }
13902 return Register();
13903}
13904
13905Register fastEmit_ISD_UMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13906 if (RetVT.SimpleTy != MVT::v4i16)
13907 return Register();
13908 if ((Subtarget->isNeonAvailable())) {
13909 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13910 }
13911 return Register();
13912}
13913
13914Register fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13915 if (RetVT.SimpleTy != MVT::v8i16)
13916 return Register();
13917 if ((Subtarget->isNeonAvailable())) {
13918 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13919 }
13920 return Register();
13921}
13922
13923Register fastEmit_ISD_UMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13924 if (RetVT.SimpleTy != MVT::v2i32)
13925 return Register();
13926 if ((Subtarget->isNeonAvailable())) {
13927 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13928 }
13929 return Register();
13930}
13931
13932Register fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13933 if (RetVT.SimpleTy != MVT::v4i32)
13934 return Register();
13935 if ((Subtarget->isNeonAvailable())) {
13936 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13937 }
13938 return Register();
13939}
13940
13941Register fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13942 switch (VT.SimpleTy) {
13943 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_rr(RetVT, Op0, Op1);
13944 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_rr(RetVT, Op0, Op1);
13945 case MVT::v8i8: return fastEmit_ISD_UMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
13946 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
13947 case MVT::v4i16: return fastEmit_ISD_UMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
13948 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
13949 case MVT::v2i32: return fastEmit_ISD_UMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
13950 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
13951 default: return Register();
13952 }
13953}
13954
13955// FastEmit functions for ISD::UMIN.
13956
13957Register fastEmit_ISD_UMIN_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13958 if (RetVT.SimpleTy != MVT::i32)
13959 return Register();
13960 if ((Subtarget->hasCSSC())) {
13961 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13962 }
13963 return Register();
13964}
13965
13966Register fastEmit_ISD_UMIN_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13967 if (RetVT.SimpleTy != MVT::i64)
13968 return Register();
13969 if ((Subtarget->hasCSSC())) {
13970 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13971 }
13972 return Register();
13973}
13974
13975Register fastEmit_ISD_UMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13976 if (RetVT.SimpleTy != MVT::v8i8)
13977 return Register();
13978 if ((Subtarget->isNeonAvailable())) {
13979 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13980 }
13981 return Register();
13982}
13983
13984Register fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13985 if (RetVT.SimpleTy != MVT::v16i8)
13986 return Register();
13987 if ((Subtarget->isNeonAvailable())) {
13988 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13989 }
13990 return Register();
13991}
13992
13993Register fastEmit_ISD_UMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13994 if (RetVT.SimpleTy != MVT::v4i16)
13995 return Register();
13996 if ((Subtarget->isNeonAvailable())) {
13997 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13998 }
13999 return Register();
14000}
14001
14002Register fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14003 if (RetVT.SimpleTy != MVT::v8i16)
14004 return Register();
14005 if ((Subtarget->isNeonAvailable())) {
14006 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
14007 }
14008 return Register();
14009}
14010
14011Register fastEmit_ISD_UMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
14012 if (RetVT.SimpleTy != MVT::v2i32)
14013 return Register();
14014 if ((Subtarget->isNeonAvailable())) {
14015 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
14016 }
14017 return Register();
14018}
14019
14020Register fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14021 if (RetVT.SimpleTy != MVT::v4i32)
14022 return Register();
14023 if ((Subtarget->isNeonAvailable())) {
14024 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
14025 }
14026 return Register();
14027}
14028
14029Register fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14030 switch (VT.SimpleTy) {
14031 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_rr(RetVT, Op0, Op1);
14032 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_rr(RetVT, Op0, Op1);
14033 case MVT::v8i8: return fastEmit_ISD_UMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
14034 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
14035 case MVT::v4i16: return fastEmit_ISD_UMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
14036 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
14037 case MVT::v2i32: return fastEmit_ISD_UMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
14038 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
14039 default: return Register();
14040 }
14041}
14042
14043// FastEmit functions for ISD::USUBSAT.
14044
14045Register fastEmit_ISD_USUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
14046 if (RetVT.SimpleTy != MVT::v8i8)
14047 return Register();
14048 if ((Subtarget->isNeonAvailable())) {
14049 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14050 }
14051 return Register();
14052}
14053
14054Register fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14055 if (RetVT.SimpleTy != MVT::v16i8)
14056 return Register();
14057 if ((Subtarget->isNeonAvailable())) {
14058 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14059 }
14060 return Register();
14061}
14062
14063Register fastEmit_ISD_USUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
14064 if (RetVT.SimpleTy != MVT::v4i16)
14065 return Register();
14066 if ((Subtarget->isNeonAvailable())) {
14067 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
14068 }
14069 return Register();
14070}
14071
14072Register fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14073 if (RetVT.SimpleTy != MVT::v8i16)
14074 return Register();
14075 if ((Subtarget->isNeonAvailable())) {
14076 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
14077 }
14078 return Register();
14079}
14080
14081Register fastEmit_ISD_USUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
14082 if (RetVT.SimpleTy != MVT::v2i32)
14083 return Register();
14084 if ((Subtarget->isNeonAvailable())) {
14085 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
14086 }
14087 return Register();
14088}
14089
14090Register fastEmit_ISD_USUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14091 if (RetVT.SimpleTy != MVT::v4i32)
14092 return Register();
14093 if ((Subtarget->isNeonAvailable())) {
14094 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
14095 }
14096 return Register();
14097}
14098
14099Register fastEmit_ISD_USUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
14100 if (RetVT.SimpleTy != MVT::v1i64)
14101 return Register();
14102 if ((Subtarget->isNeonAvailable())) {
14103 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
14104 }
14105 return Register();
14106}
14107
14108Register fastEmit_ISD_USUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14109 if (RetVT.SimpleTy != MVT::v2i64)
14110 return Register();
14111 if ((Subtarget->isNeonAvailable())) {
14112 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
14113 }
14114 return Register();
14115}
14116
14117Register fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14118 if (RetVT.SimpleTy != MVT::nxv16i8)
14119 return Register();
14120 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14121 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
14122 }
14123 return Register();
14124}
14125
14126Register fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14127 if (RetVT.SimpleTy != MVT::nxv8i16)
14128 return Register();
14129 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14130 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
14131 }
14132 return Register();
14133}
14134
14135Register fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14136 if (RetVT.SimpleTy != MVT::nxv4i32)
14137 return Register();
14138 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14139 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
14140 }
14141 return Register();
14142}
14143
14144Register fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14145 if (RetVT.SimpleTy != MVT::nxv2i64)
14146 return Register();
14147 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14148 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
14149 }
14150 return Register();
14151}
14152
14153Register fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14154 switch (VT.SimpleTy) {
14155 case MVT::v8i8: return fastEmit_ISD_USUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
14156 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
14157 case MVT::v4i16: return fastEmit_ISD_USUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
14158 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
14159 case MVT::v2i32: return fastEmit_ISD_USUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
14160 case MVT::v4i32: return fastEmit_ISD_USUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
14161 case MVT::v1i64: return fastEmit_ISD_USUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
14162 case MVT::v2i64: return fastEmit_ISD_USUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
14163 case MVT::nxv16i8: return fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
14164 case MVT::nxv8i16: return fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
14165 case MVT::nxv4i32: return fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
14166 case MVT::nxv2i64: return fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
14167 default: return Register();
14168 }
14169}
14170
14171// FastEmit functions for ISD::XOR.
14172
14173Register fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
14174 if (RetVT.SimpleTy != MVT::i32)
14175 return Register();
14176 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
14177}
14178
14179Register fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
14180 if (RetVT.SimpleTy != MVT::i64)
14181 return Register();
14182 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
14183}
14184
14185Register fastEmit_ISD_XOR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
14186 if (RetVT.SimpleTy != MVT::v8i8)
14187 return Register();
14188 if ((Subtarget->isNeonAvailable())) {
14189 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14190 }
14191 return Register();
14192}
14193
14194Register fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14195 if (RetVT.SimpleTy != MVT::v16i8)
14196 return Register();
14197 if ((Subtarget->isNeonAvailable())) {
14198 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14199 }
14200 return Register();
14201}
14202
14203Register fastEmit_ISD_XOR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
14204 if (RetVT.SimpleTy != MVT::v4i16)
14205 return Register();
14206 if ((Subtarget->isNeonAvailable())) {
14207 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14208 }
14209 return Register();
14210}
14211
14212Register fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14213 if (RetVT.SimpleTy != MVT::v8i16)
14214 return Register();
14215 if ((Subtarget->isNeonAvailable())) {
14216 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14217 }
14218 return Register();
14219}
14220
14221Register fastEmit_ISD_XOR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
14222 if (RetVT.SimpleTy != MVT::v2i32)
14223 return Register();
14224 if ((Subtarget->isNeonAvailable())) {
14225 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14226 }
14227 return Register();
14228}
14229
14230Register fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14231 if (RetVT.SimpleTy != MVT::v4i32)
14232 return Register();
14233 if ((Subtarget->isNeonAvailable())) {
14234 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14235 }
14236 return Register();
14237}
14238
14239Register fastEmit_ISD_XOR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
14240 if (RetVT.SimpleTy != MVT::v1i64)
14241 return Register();
14242 if ((Subtarget->isNeonAvailable())) {
14243 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14244 }
14245 return Register();
14246}
14247
14248Register fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14249 if (RetVT.SimpleTy != MVT::v2i64)
14250 return Register();
14251 if ((Subtarget->isNeonAvailable())) {
14252 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14253 }
14254 return Register();
14255}
14256
14257Register fastEmit_ISD_XOR_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14258 if (RetVT.SimpleTy != MVT::nxv16i8)
14259 return Register();
14260 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14261 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14262 }
14263 return Register();
14264}
14265
14266Register fastEmit_ISD_XOR_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14267 if (RetVT.SimpleTy != MVT::nxv8i16)
14268 return Register();
14269 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14270 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14271 }
14272 return Register();
14273}
14274
14275Register fastEmit_ISD_XOR_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14276 if (RetVT.SimpleTy != MVT::nxv4i32)
14277 return Register();
14278 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14279 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14280 }
14281 return Register();
14282}
14283
14284Register fastEmit_ISD_XOR_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14285 if (RetVT.SimpleTy != MVT::nxv2i64)
14286 return Register();
14287 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14288 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14289 }
14290 return Register();
14291}
14292
14293Register fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14294 switch (VT.SimpleTy) {
14295 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
14296 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op1);
14297 case MVT::v8i8: return fastEmit_ISD_XOR_MVT_v8i8_rr(RetVT, Op0, Op1);
14298 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
14299 case MVT::v4i16: return fastEmit_ISD_XOR_MVT_v4i16_rr(RetVT, Op0, Op1);
14300 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
14301 case MVT::v2i32: return fastEmit_ISD_XOR_MVT_v2i32_rr(RetVT, Op0, Op1);
14302 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
14303 case MVT::v1i64: return fastEmit_ISD_XOR_MVT_v1i64_rr(RetVT, Op0, Op1);
14304 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
14305 case MVT::nxv16i8: return fastEmit_ISD_XOR_MVT_nxv16i8_rr(RetVT, Op0, Op1);
14306 case MVT::nxv8i16: return fastEmit_ISD_XOR_MVT_nxv8i16_rr(RetVT, Op0, Op1);
14307 case MVT::nxv4i32: return fastEmit_ISD_XOR_MVT_nxv4i32_rr(RetVT, Op0, Op1);
14308 case MVT::nxv2i64: return fastEmit_ISD_XOR_MVT_nxv2i64_rr(RetVT, Op0, Op1);
14309 default: return Register();
14310 }
14311}
14312
14313// Top-level FastEmit function.
14314
14315Register fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, Register Op1) override {
14316 switch (Opcode) {
14317 case AArch64ISD::ADDP: return fastEmit_AArch64ISD_ADDP_rr(VT, RetVT, Op0, Op1);
14318 case AArch64ISD::BIC: return fastEmit_AArch64ISD_BIC_rr(VT, RetVT, Op0, Op1);
14319 case AArch64ISD::FCMEQ: return fastEmit_AArch64ISD_FCMEQ_rr(VT, RetVT, Op0, Op1);
14320 case AArch64ISD::FCMGE: return fastEmit_AArch64ISD_FCMGE_rr(VT, RetVT, Op0, Op1);
14321 case AArch64ISD::FCMGT: return fastEmit_AArch64ISD_FCMGT_rr(VT, RetVT, Op0, Op1);
14322 case AArch64ISD::FCMP: return fastEmit_AArch64ISD_FCMP_rr(VT, RetVT, Op0, Op1);
14323 case AArch64ISD::FRECPS: return fastEmit_AArch64ISD_FRECPS_rr(VT, RetVT, Op0, Op1);
14324 case AArch64ISD::FRSQRTS: return fastEmit_AArch64ISD_FRSQRTS_rr(VT, RetVT, Op0, Op1);
14325 case AArch64ISD::INIT_TPIDR2OBJ: return fastEmit_AArch64ISD_INIT_TPIDR2OBJ_rr(VT, RetVT, Op0, Op1);
14326 case AArch64ISD::PMULL: return fastEmit_AArch64ISD_PMULL_rr(VT, RetVT, Op0, Op1);
14327 case AArch64ISD::PTEST: return fastEmit_AArch64ISD_PTEST_rr(VT, RetVT, Op0, Op1);
14328 case AArch64ISD::PTEST_ANY: return fastEmit_AArch64ISD_PTEST_ANY_rr(VT, RetVT, Op0, Op1);
14329 case AArch64ISD::PTEST_FIRST: return fastEmit_AArch64ISD_PTEST_FIRST_rr(VT, RetVT, Op0, Op1);
14330 case AArch64ISD::SMULL: return fastEmit_AArch64ISD_SMULL_rr(VT, RetVT, Op0, Op1);
14331 case AArch64ISD::SQADD: return fastEmit_AArch64ISD_SQADD_rr(VT, RetVT, Op0, Op1);
14332 case AArch64ISD::SQDMULH: return fastEmit_AArch64ISD_SQDMULH_rr(VT, RetVT, Op0, Op1);
14333 case AArch64ISD::SQDMULL: return fastEmit_AArch64ISD_SQDMULL_rr(VT, RetVT, Op0, Op1);
14334 case AArch64ISD::SQRDMULH: return fastEmit_AArch64ISD_SQRDMULH_rr(VT, RetVT, Op0, Op1);
14335 case AArch64ISD::SQRSHL: return fastEmit_AArch64ISD_SQRSHL_rr(VT, RetVT, Op0, Op1);
14336 case AArch64ISD::SQSHL: return fastEmit_AArch64ISD_SQSHL_rr(VT, RetVT, Op0, Op1);
14337 case AArch64ISD::SQSUB: return fastEmit_AArch64ISD_SQSUB_rr(VT, RetVT, Op0, Op1);
14338 case AArch64ISD::STRICT_FCMP: return fastEmit_AArch64ISD_STRICT_FCMP_rr(VT, RetVT, Op0, Op1);
14339 case AArch64ISD::STRICT_FCMPE: return fastEmit_AArch64ISD_STRICT_FCMPE_rr(VT, RetVT, Op0, Op1);
14340 case AArch64ISD::SUQADD: return fastEmit_AArch64ISD_SUQADD_rr(VT, RetVT, Op0, Op1);
14341 case AArch64ISD::TBL: return fastEmit_AArch64ISD_TBL_rr(VT, RetVT, Op0, Op1);
14342 case AArch64ISD::TRN1: return fastEmit_AArch64ISD_TRN1_rr(VT, RetVT, Op0, Op1);
14343 case AArch64ISD::TRN2: return fastEmit_AArch64ISD_TRN2_rr(VT, RetVT, Op0, Op1);
14344 case AArch64ISD::UMULL: return fastEmit_AArch64ISD_UMULL_rr(VT, RetVT, Op0, Op1);
14345 case AArch64ISD::UQADD: return fastEmit_AArch64ISD_UQADD_rr(VT, RetVT, Op0, Op1);
14346 case AArch64ISD::UQRSHL: return fastEmit_AArch64ISD_UQRSHL_rr(VT, RetVT, Op0, Op1);
14347 case AArch64ISD::UQSHL: return fastEmit_AArch64ISD_UQSHL_rr(VT, RetVT, Op0, Op1);
14348 case AArch64ISD::UQSUB: return fastEmit_AArch64ISD_UQSUB_rr(VT, RetVT, Op0, Op1);
14349 case AArch64ISD::USQADD: return fastEmit_AArch64ISD_USQADD_rr(VT, RetVT, Op0, Op1);
14350 case AArch64ISD::UZP1: return fastEmit_AArch64ISD_UZP1_rr(VT, RetVT, Op0, Op1);
14351 case AArch64ISD::UZP2: return fastEmit_AArch64ISD_UZP2_rr(VT, RetVT, Op0, Op1);
14352 case AArch64ISD::ZIP1: return fastEmit_AArch64ISD_ZIP1_rr(VT, RetVT, Op0, Op1);
14353 case AArch64ISD::ZIP2: return fastEmit_AArch64ISD_ZIP2_rr(VT, RetVT, Op0, Op1);
14354 case ISD::ABDS: return fastEmit_ISD_ABDS_rr(VT, RetVT, Op0, Op1);
14355 case ISD::ABDU: return fastEmit_ISD_ABDU_rr(VT, RetVT, Op0, Op1);
14356 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
14357 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
14358 case ISD::AVGCEILS: return fastEmit_ISD_AVGCEILS_rr(VT, RetVT, Op0, Op1);
14359 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
14360 case ISD::AVGFLOORS: return fastEmit_ISD_AVGFLOORS_rr(VT, RetVT, Op0, Op1);
14361 case ISD::AVGFLOORU: return fastEmit_ISD_AVGFLOORU_rr(VT, RetVT, Op0, Op1);
14362 case ISD::CLMUL: return fastEmit_ISD_CLMUL_rr(VT, RetVT, Op0, Op1);
14363 case ISD::CONCAT_VECTORS: return fastEmit_ISD_CONCAT_VECTORS_rr(VT, RetVT, Op0, Op1);
14364 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
14365 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
14366 case ISD::FMAXIMUM: return fastEmit_ISD_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
14367 case ISD::FMAXNUM: return fastEmit_ISD_FMAXNUM_rr(VT, RetVT, Op0, Op1);
14368 case ISD::FMAXNUM_IEEE: return fastEmit_ISD_FMAXNUM_IEEE_rr(VT, RetVT, Op0, Op1);
14369 case ISD::FMINIMUM: return fastEmit_ISD_FMINIMUM_rr(VT, RetVT, Op0, Op1);
14370 case ISD::FMINNUM: return fastEmit_ISD_FMINNUM_rr(VT, RetVT, Op0, Op1);
14371 case ISD::FMINNUM_IEEE: return fastEmit_ISD_FMINNUM_IEEE_rr(VT, RetVT, Op0, Op1);
14372 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
14373 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
14374 case ISD::GET_ACTIVE_LANE_MASK: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_rr(VT, RetVT, Op0, Op1);
14375 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
14376 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
14377 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
14378 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
14379 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
14380 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
14381 case ISD::SDIV: return fastEmit_ISD_SDIV_rr(VT, RetVT, Op0, Op1);
14382 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
14383 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
14384 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
14385 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
14386 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
14387 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
14388 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1);
14389 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1);
14390 case ISD::STRICT_FMAXIMUM: return fastEmit_ISD_STRICT_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
14391 case ISD::STRICT_FMAXNUM: return fastEmit_ISD_STRICT_FMAXNUM_rr(VT, RetVT, Op0, Op1);
14392 case ISD::STRICT_FMINIMUM: return fastEmit_ISD_STRICT_FMINIMUM_rr(VT, RetVT, Op0, Op1);
14393 case ISD::STRICT_FMINNUM: return fastEmit_ISD_STRICT_FMINNUM_rr(VT, RetVT, Op0, Op1);
14394 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1);
14395 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1);
14396 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
14397 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
14398 case ISD::UDIV: return fastEmit_ISD_UDIV_rr(VT, RetVT, Op0, Op1);
14399 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
14400 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
14401 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
14402 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
14403 default: return Register();
14404 }
14405}
14406
14407// FastEmit functions for AArch64ISD::DUPLANE64.
14408
14409Register fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14410 if (RetVT.SimpleTy != MVT::v2i64)
14411 return Register();
14412 if ((Subtarget->isNeonAvailable())) {
14413 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i64lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14414 }
14415 return Register();
14416}
14417
14418Register fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14419 if (RetVT.SimpleTy != MVT::v2f64)
14420 return Register();
14421 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i64lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14422}
14423
14424Register fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14425 switch (VT.SimpleTy) {
14426 case MVT::v2i64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14427 case MVT::v2f64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14428 default: return Register();
14429 }
14430}
14431
14432// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14433
14434Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14435 if (RetVT.SimpleTy != MVT::i64)
14436 return Register();
14437 if ((Subtarget->isNeonAvailable())) {
14438 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi64, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14439 }
14440 return Register();
14441}
14442
14443Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14444 if (RetVT.SimpleTy != MVT::f64)
14445 return Register();
14446 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi64, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14447}
14448
14449Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14450 switch (VT.SimpleTy) {
14451 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14452 case MVT::v2f64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14453 default: return Register();
14454 }
14455}
14456
14457// Top-level FastEmit function.
14458
14459Register fastEmit_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14460 switch (Opcode) {
14461 case AArch64ISD::DUPLANE64: return fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1);
14462 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1);
14463 default: return Register();
14464 }
14465}
14466
14467// FastEmit functions for AArch64ISD::DUPLANE32.
14468
14469Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14470 if ((Subtarget->isNeonAvailable())) {
14471 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i32lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14472 }
14473 return Register();
14474}
14475
14476Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14477 if ((Subtarget->isNeonAvailable())) {
14478 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i32lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14479 }
14480 return Register();
14481}
14482
14483Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14484switch (RetVT.SimpleTy) {
14485 case MVT::v2i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Op0, imm1);
14486 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Op0, imm1);
14487 default: return Register();
14488}
14489}
14490
14491Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14492 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i32lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14493}
14494
14495Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14496 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i32lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14497}
14498
14499Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14500switch (RetVT.SimpleTy) {
14501 case MVT::v2f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Op0, imm1);
14502 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Op0, imm1);
14503 default: return Register();
14504}
14505}
14506
14507Register fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14508 switch (VT.SimpleTy) {
14509 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14510 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14511 default: return Register();
14512 }
14513}
14514
14515// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14516
14517Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14518 if (RetVT.SimpleTy != MVT::i32)
14519 return Register();
14520 if ((Subtarget->isNeonAvailable())) {
14521 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi32, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14522 }
14523 return Register();
14524}
14525
14526Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14527 if (RetVT.SimpleTy != MVT::f32)
14528 return Register();
14529 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi32, RC: &AArch64::FPR32RegClass, Op0, Imm: imm1);
14530}
14531
14532Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14533 switch (VT.SimpleTy) {
14534 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14535 case MVT::v4f32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14536 default: return Register();
14537 }
14538}
14539
14540// Top-level FastEmit function.
14541
14542Register fastEmit_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14543 switch (Opcode) {
14544 case AArch64ISD::DUPLANE32: return fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1);
14545 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1);
14546 default: return Register();
14547 }
14548}
14549
14550// FastEmit functions for AArch64ISD::DUPLANE16.
14551
14552Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14553 if ((Subtarget->isNeonAvailable())) {
14554 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14555 }
14556 return Register();
14557}
14558
14559Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14560 if ((Subtarget->isNeonAvailable())) {
14561 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14562 }
14563 return Register();
14564}
14565
14566Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14567switch (RetVT.SimpleTy) {
14568 case MVT::v4i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Op0, imm1);
14569 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Op0, imm1);
14570 default: return Register();
14571}
14572}
14573
14574Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14575 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14576}
14577
14578Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14579 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14580}
14581
14582Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14583switch (RetVT.SimpleTy) {
14584 case MVT::v4f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Op0, imm1);
14585 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Op0, imm1);
14586 default: return Register();
14587}
14588}
14589
14590Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14591 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14592}
14593
14594Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14595 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14596}
14597
14598Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14599switch (RetVT.SimpleTy) {
14600 case MVT::v4bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(Op0, imm1);
14601 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(Op0, imm1);
14602 default: return Register();
14603}
14604}
14605
14606Register fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14607 switch (VT.SimpleTy) {
14608 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14609 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14610 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14611 default: return Register();
14612 }
14613}
14614
14615// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14616
14617Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14618 if (RetVT.SimpleTy != MVT::i32)
14619 return Register();
14620 if ((Subtarget->isNeonAvailable())) {
14621 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi16, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14622 }
14623 return Register();
14624}
14625
14626Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14627 if (RetVT.SimpleTy != MVT::f16)
14628 return Register();
14629 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi16, RC: &AArch64::FPR16RegClass, Op0, Imm: imm1);
14630}
14631
14632Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14633 if (RetVT.SimpleTy != MVT::bf16)
14634 return Register();
14635 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi16, RC: &AArch64::FPR16RegClass, Op0, Imm: imm1);
14636}
14637
14638Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14639 switch (VT.SimpleTy) {
14640 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14641 case MVT::v8f16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14642 case MVT::v8bf16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14643 default: return Register();
14644 }
14645}
14646
14647// Top-level FastEmit function.
14648
14649Register fastEmit_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14650 switch (Opcode) {
14651 case AArch64ISD::DUPLANE16: return fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1);
14652 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1);
14653 default: return Register();
14654 }
14655}
14656
14657// FastEmit functions for AArch64ISD::DUPLANE8.
14658
14659Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Register Op0, uint64_t imm1) {
14660 if ((Subtarget->isNeonAvailable())) {
14661 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i8lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14662 }
14663 return Register();
14664}
14665
14666Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Register Op0, uint64_t imm1) {
14667 if ((Subtarget->isNeonAvailable())) {
14668 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv16i8lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14669 }
14670 return Register();
14671}
14672
14673Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, Register Op0, uint64_t imm1) {
14674switch (RetVT.SimpleTy) {
14675 case MVT::v8i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Op0, imm1);
14676 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Op0, imm1);
14677 default: return Register();
14678}
14679}
14680
14681Register fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14682 switch (VT.SimpleTy) {
14683 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1);
14684 default: return Register();
14685 }
14686}
14687
14688// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14689
14690Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, Register Op0, uint64_t imm1) {
14691 if (RetVT.SimpleTy != MVT::i32)
14692 return Register();
14693 if ((Subtarget->isNeonAvailable())) {
14694 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi8, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14695 }
14696 return Register();
14697}
14698
14699Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14700 switch (VT.SimpleTy) {
14701 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1);
14702 default: return Register();
14703 }
14704}
14705
14706// Top-level FastEmit function.
14707
14708Register fastEmit_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14709 switch (Opcode) {
14710 case AArch64ISD::DUPLANE8: return fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1);
14711 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1);
14712 default: return Register();
14713 }
14714}
14715
14716// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14717
14718Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14719 if (RetVT.SimpleTy != MVT::i32)
14720 return Register();
14721 if ((Subtarget->hasNEON())) {
14722 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi8_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14723 }
14724 return Register();
14725}
14726
14727Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14728 if (RetVT.SimpleTy != MVT::i32)
14729 return Register();
14730 if ((Subtarget->hasNEON())) {
14731 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi16_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14732 }
14733 return Register();
14734}
14735
14736Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14737 if (RetVT.SimpleTy != MVT::i32)
14738 return Register();
14739 if ((Subtarget->hasNEON())) {
14740 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi32_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14741 }
14742 return Register();
14743}
14744
14745Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14746 if (RetVT.SimpleTy != MVT::i64)
14747 return Register();
14748 if ((Subtarget->hasNEON())) {
14749 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi64_idx0, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14750 }
14751 return Register();
14752}
14753
14754Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14755 switch (VT.SimpleTy) {
14756 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14757 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14758 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14759 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14760 default: return Register();
14761 }
14762}
14763
14764// Top-level FastEmit function.
14765
14766Register fastEmit_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14767 switch (Opcode) {
14768 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(VT, RetVT, Op0, imm1);
14769 default: return Register();
14770 }
14771}
14772
14773// FastEmit functions for ISD::SMAX.
14774
14775Register fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14776 if (RetVT.SimpleTy != MVT::i32)
14777 return Register();
14778 if ((Subtarget->hasCSSC())) {
14779 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMAXWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14780 }
14781 return Register();
14782}
14783
14784Register fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14785 switch (VT.SimpleTy) {
14786 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1);
14787 default: return Register();
14788 }
14789}
14790
14791// FastEmit functions for ISD::SMIN.
14792
14793Register fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14794 if (RetVT.SimpleTy != MVT::i32)
14795 return Register();
14796 if ((Subtarget->hasCSSC())) {
14797 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMINWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14798 }
14799 return Register();
14800}
14801
14802Register fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14803 switch (VT.SimpleTy) {
14804 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1);
14805 default: return Register();
14806 }
14807}
14808
14809// Top-level FastEmit function.
14810
14811Register fastEmit_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14812 switch (Opcode) {
14813 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1);
14814 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1);
14815 default: return Register();
14816 }
14817}
14818
14819// FastEmit functions for ISD::SMAX.
14820
14821Register fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14822 if (RetVT.SimpleTy != MVT::i64)
14823 return Register();
14824 if ((Subtarget->hasCSSC())) {
14825 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMAXXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14826 }
14827 return Register();
14828}
14829
14830Register fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14831 switch (VT.SimpleTy) {
14832 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1);
14833 default: return Register();
14834 }
14835}
14836
14837// FastEmit functions for ISD::SMIN.
14838
14839Register fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14840 if (RetVT.SimpleTy != MVT::i64)
14841 return Register();
14842 if ((Subtarget->hasCSSC())) {
14843 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMINXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14844 }
14845 return Register();
14846}
14847
14848Register fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14849 switch (VT.SimpleTy) {
14850 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1);
14851 default: return Register();
14852 }
14853}
14854
14855// Top-level FastEmit function.
14856
14857Register fastEmit_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14858 switch (Opcode) {
14859 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1);
14860 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1);
14861 default: return Register();
14862 }
14863}
14864
14865// FastEmit functions for ISD::UMAX.
14866
14867Register fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14868 if (RetVT.SimpleTy != MVT::i32)
14869 return Register();
14870 if ((Subtarget->hasCSSC())) {
14871 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMAXWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14872 }
14873 return Register();
14874}
14875
14876Register fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14877 switch (VT.SimpleTy) {
14878 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1);
14879 default: return Register();
14880 }
14881}
14882
14883// FastEmit functions for ISD::UMIN.
14884
14885Register fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14886 if (RetVT.SimpleTy != MVT::i32)
14887 return Register();
14888 if ((Subtarget->hasCSSC())) {
14889 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMINWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14890 }
14891 return Register();
14892}
14893
14894Register fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14895 switch (VT.SimpleTy) {
14896 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1);
14897 default: return Register();
14898 }
14899}
14900
14901// Top-level FastEmit function.
14902
14903Register fastEmit_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14904 switch (Opcode) {
14905 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1);
14906 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1);
14907 default: return Register();
14908 }
14909}
14910
14911// FastEmit functions for ISD::UMAX.
14912
14913Register fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14914 if (RetVT.SimpleTy != MVT::i64)
14915 return Register();
14916 if ((Subtarget->hasCSSC())) {
14917 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMAXXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14918 }
14919 return Register();
14920}
14921
14922Register fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14923 switch (VT.SimpleTy) {
14924 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1);
14925 default: return Register();
14926 }
14927}
14928
14929// FastEmit functions for ISD::UMIN.
14930
14931Register fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14932 if (RetVT.SimpleTy != MVT::i64)
14933 return Register();
14934 if ((Subtarget->hasCSSC())) {
14935 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMINXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14936 }
14937 return Register();
14938}
14939
14940Register fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14941 switch (VT.SimpleTy) {
14942 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1);
14943 default: return Register();
14944 }
14945}
14946
14947// Top-level FastEmit function.
14948
14949Register fastEmit_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14950 switch (Opcode) {
14951 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1);
14952 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1);
14953 default: return Register();
14954 }
14955}
14956
14957// FastEmit functions for ISD::Constant.
14958
14959Register fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
14960 if (RetVT.SimpleTy != MVT::i32)
14961 return Register();
14962 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVi32imm, RC: &AArch64::GPR32RegClass, Imm: imm0);
14963}
14964
14965Register fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) {
14966 if (RetVT.SimpleTy != MVT::i64)
14967 return Register();
14968 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVi64imm, RC: &AArch64::GPR64RegClass, Imm: imm0);
14969}
14970
14971Register fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
14972 switch (VT.SimpleTy) {
14973 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
14974 case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0);
14975 default: return Register();
14976 }
14977}
14978
14979// Top-level FastEmit function.
14980
14981Register fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
14982 if (VT == MVT::i32 && Predicate_imm0_255(Imm: imm0))
14983 if (Register Reg = fastEmit_i_Predicate_imm0_255(VT, RetVT, Opcode, imm0))
14984 return Reg;
14985
14986 if (VT == MVT::i32 && Predicate_simm6_32b(Imm: imm0))
14987 if (Register Reg = fastEmit_i_Predicate_simm6_32b(VT, RetVT, Opcode, imm0))
14988 return Reg;
14989
14990 switch (Opcode) {
14991 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
14992 default: return Register();
14993 }
14994}
14995
14996// FastEmit functions for AArch64ISD::FMOV.
14997
14998Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(uint64_t imm0) {
14999 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
15000 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv4f16_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
15001 }
15002 return Register();
15003}
15004
15005Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(uint64_t imm0) {
15006 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
15007 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv8f16_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
15008 }
15009 return Register();
15010}
15011
15012Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(uint64_t imm0) {
15013 if ((Subtarget->isNeonAvailable())) {
15014 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv2f32_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
15015 }
15016 return Register();
15017}
15018
15019Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(uint64_t imm0) {
15020 if ((Subtarget->isNeonAvailable())) {
15021 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv4f32_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
15022 }
15023 return Register();
15024}
15025
15026Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(uint64_t imm0) {
15027 if ((Subtarget->isNeonAvailable())) {
15028 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv2f64_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
15029 }
15030 return Register();
15031}
15032
15033Register fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
15034switch (RetVT.SimpleTy) {
15035 case MVT::v4f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(imm0);
15036 case MVT::v8f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(imm0);
15037 case MVT::v2f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(imm0);
15038 case MVT::v4f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(imm0);
15039 case MVT::v2f64: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(imm0);
15040 default: return Register();
15041}
15042}
15043
15044Register fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
15045 switch (VT.SimpleTy) {
15046 case MVT::i32: return fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
15047 default: return Register();
15048 }
15049}
15050
15051// FastEmit functions for AArch64ISD::MOVI.
15052
15053Register fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(uint64_t imm0) {
15054 if ((Subtarget->isNeonAvailable())) {
15055 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv8b_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
15056 }
15057 return Register();
15058}
15059
15060Register fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(uint64_t imm0) {
15061 if ((Subtarget->isNeonAvailable())) {
15062 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv16b_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
15063 }
15064 return Register();
15065}
15066
15067Register fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
15068switch (RetVT.SimpleTy) {
15069 case MVT::v8i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(imm0);
15070 case MVT::v16i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(imm0);
15071 default: return Register();
15072}
15073}
15074
15075Register fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
15076 switch (VT.SimpleTy) {
15077 case MVT::i32: return fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
15078 default: return Register();
15079 }
15080}
15081
15082// FastEmit functions for AArch64ISD::MOVIedit.
15083
15084Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(uint64_t imm0) {
15085 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVID, RC: &AArch64::FPR64RegClass, Imm: imm0);
15086}
15087
15088Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(uint64_t imm0) {
15089 if ((Subtarget->isNeonAvailable())) {
15090 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv2d_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
15091 }
15092 return Register();
15093}
15094
15095Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
15096switch (RetVT.SimpleTy) {
15097 case MVT::f64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(imm0);
15098 case MVT::v2i64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(imm0);
15099 default: return Register();
15100}
15101}
15102
15103Register fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
15104 switch (VT.SimpleTy) {
15105 case MVT::i32: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
15106 default: return Register();
15107 }
15108}
15109
15110// Top-level FastEmit function.
15111
15112Register fastEmit_i_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
15113 switch (Opcode) {
15114 case AArch64ISD::FMOV: return fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(VT, RetVT, imm0);
15115 case AArch64ISD::MOVI: return fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(VT, RetVT, imm0);
15116 case AArch64ISD::MOVIedit: return fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(VT, RetVT, imm0);
15117 default: return Register();
15118 }
15119}
15120
15121// FastEmit functions for AArch64ISD::RDSVL.
15122
15123Register fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(MVT RetVT, uint64_t imm0) {
15124 if (RetVT.SimpleTy != MVT::i64)
15125 return Register();
15126 if ((Subtarget->hasSME())) {
15127 return fastEmitInst_i(MachineInstOpcode: AArch64::RDSVLI_XI, RC: &AArch64::GPR64RegClass, Imm: imm0);
15128 }
15129 return Register();
15130}
15131
15132Register fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(MVT VT, MVT RetVT, uint64_t imm0) {
15133 switch (VT.SimpleTy) {
15134 case MVT::i32: return fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(RetVT, imm0);
15135 default: return Register();
15136 }
15137}
15138
15139// Top-level FastEmit function.
15140
15141Register fastEmit_i_Predicate_simm6_32b(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
15142 switch (Opcode) {
15143 case AArch64ISD::RDSVL: return fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(VT, RetVT, imm0);
15144 default: return Register();
15145 }
15146}
15147
15148