1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the AArch64 target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_imm0_31(int64_t Imm) {
12
13 return ((uint64_t)Imm) < 32;
14
15}
16static bool Predicate_imm0_63(int64_t Imm) {
17
18 return ((uint64_t)Imm) < 64;
19
20}
21static bool Predicate_imm32_0_31(int64_t Imm) {
22
23 return ((uint64_t)Imm) < 32;
24
25}
26static bool Predicate_tbz_imm0_31_diag(int64_t Imm) {
27
28 return (((uint32_t)Imm) < 32);
29
30}
31static bool Predicate_tbz_imm32_63(int64_t Imm) {
32
33 return (((uint32_t)Imm) > 31) && (((uint32_t)Imm) < 64);
34
35}
36static bool Predicate_VectorIndexD(int64_t Imm) {
37 return ((uint64_t)Imm) < 2;
38}
39static bool Predicate_VectorIndexS(int64_t Imm) {
40 return ((uint64_t)Imm) < 4;
41}
42static bool Predicate_VectorIndexH(int64_t Imm) {
43 return ((uint64_t)Imm) < 8;
44}
45static bool Predicate_VectorIndexB(int64_t Imm) {
46 return ((uint64_t)Imm) < 16;
47}
48static bool Predicate_VectorIndex0(int64_t Imm) {
49 return ((uint64_t)Imm) == 0;
50}
51static bool Predicate_imm0_255(int64_t Imm) {
52
53 return ((uint32_t)Imm) < 256;
54
55}
56static bool Predicate_simm8_32b(int64_t Imm) {
57 return Imm >= -128 && Imm < 128;
58}
59static bool Predicate_simm8_64b(int64_t Imm) {
60 return Imm >= -128 && Imm < 128;
61}
62static bool Predicate_uimm8_32b(int64_t Imm) {
63 return Imm >= 0 && Imm < 256;
64}
65static bool Predicate_uimm8_64b(int64_t Imm) {
66 return Imm >= 0 && Imm < 256;
67}
68static bool Predicate_simm6_32b(int64_t Imm) {
69 return Imm >= -32 && Imm < 32;
70}
71
72
73// FastEmit functions for AArch64ISD::ENTRY_PSTATE_SM.
74
75Register fastEmit_AArch64ISD_ENTRY_PSTATE_SM_MVT_i64_(MVT RetVT) {
76 if (RetVT.SimpleTy != MVT::i64)
77 return Register();
78 return fastEmitInst_(MachineInstOpcode: AArch64::EntryPStateSM, RC: &AArch64::GPR64RegClass);
79}
80
81Register fastEmit_AArch64ISD_ENTRY_PSTATE_SM_(MVT VT, MVT RetVT) {
82 switch (VT.SimpleTy) {
83 case MVT::i64: return fastEmit_AArch64ISD_ENTRY_PSTATE_SM_MVT_i64_(RetVT);
84 default: return Register();
85 }
86}
87
88// FastEmit functions for AArch64ISD::GET_SME_SAVE_SIZE.
89
90Register fastEmit_AArch64ISD_GET_SME_SAVE_SIZE_MVT_i64_(MVT RetVT) {
91 if (RetVT.SimpleTy != MVT::i64)
92 return Register();
93 return fastEmitInst_(MachineInstOpcode: AArch64::GetSMESaveSize, RC: &AArch64::GPR64RegClass);
94}
95
96Register fastEmit_AArch64ISD_GET_SME_SAVE_SIZE_(MVT VT, MVT RetVT) {
97 switch (VT.SimpleTy) {
98 case MVT::i64: return fastEmit_AArch64ISD_GET_SME_SAVE_SIZE_MVT_i64_(RetVT);
99 default: return Register();
100 }
101}
102
103// FastEmit functions for AArch64ISD::THREAD_POINTER.
104
105Register fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(MVT RetVT) {
106 if (RetVT.SimpleTy != MVT::i64)
107 return Register();
108 return fastEmitInst_(MachineInstOpcode: AArch64::MOVbaseTLS, RC: &AArch64::GPR64RegClass);
109}
110
111Register fastEmit_AArch64ISD_THREAD_POINTER_(MVT VT, MVT RetVT) {
112 switch (VT.SimpleTy) {
113 case MVT::i64: return fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(RetVT);
114 default: return Register();
115 }
116}
117
118// Top-level FastEmit function.
119
120Register fastEmit_(MVT VT, MVT RetVT, unsigned Opcode) override {
121 switch (Opcode) {
122 case AArch64ISD::ENTRY_PSTATE_SM: return fastEmit_AArch64ISD_ENTRY_PSTATE_SM_(VT, RetVT);
123 case AArch64ISD::GET_SME_SAVE_SIZE: return fastEmit_AArch64ISD_GET_SME_SAVE_SIZE_(VT, RetVT);
124 case AArch64ISD::THREAD_POINTER: return fastEmit_AArch64ISD_THREAD_POINTER_(VT, RetVT);
125 default: return Register();
126 }
127}
128
129// FastEmit functions for AArch64ISD::ALLOCATE_ZA_BUFFER.
130
131Register fastEmit_AArch64ISD_ALLOCATE_ZA_BUFFER_MVT_i64_r(MVT RetVT, Register Op0) {
132 if (RetVT.SimpleTy != MVT::i64)
133 return Register();
134 return fastEmitInst_r(MachineInstOpcode: AArch64::AllocateZABuffer, RC: &AArch64::GPR64spRegClass, Op0);
135}
136
137Register fastEmit_AArch64ISD_ALLOCATE_ZA_BUFFER_r(MVT VT, MVT RetVT, Register Op0) {
138 switch (VT.SimpleTy) {
139 case MVT::i64: return fastEmit_AArch64ISD_ALLOCATE_ZA_BUFFER_MVT_i64_r(RetVT, Op0);
140 default: return Register();
141 }
142}
143
144// FastEmit functions for AArch64ISD::ALLOC_SME_SAVE_BUFFER.
145
146Register fastEmit_AArch64ISD_ALLOC_SME_SAVE_BUFFER_MVT_i64_r(MVT RetVT, Register Op0) {
147 if (RetVT.SimpleTy != MVT::i64)
148 return Register();
149 return fastEmitInst_r(MachineInstOpcode: AArch64::AllocateSMESaveBuffer, RC: &AArch64::GPR64spRegClass, Op0);
150}
151
152Register fastEmit_AArch64ISD_ALLOC_SME_SAVE_BUFFER_r(MVT VT, MVT RetVT, Register Op0) {
153 switch (VT.SimpleTy) {
154 case MVT::i64: return fastEmit_AArch64ISD_ALLOC_SME_SAVE_BUFFER_MVT_i64_r(RetVT, Op0);
155 default: return Register();
156 }
157}
158
159// FastEmit functions for AArch64ISD::CALL.
160
161Register fastEmit_AArch64ISD_CALL_MVT_i64_r(MVT RetVT, Register Op0) {
162 if (RetVT.SimpleTy != MVT::isVoid)
163 return Register();
164 if (( MF->getSubtarget<AArch64Subtarget>().hardenSlsBlr() )) {
165 return fastEmitInst_r(MachineInstOpcode: AArch64::BLRNoIP, RC: &AArch64::GPR64noipRegClass, Op0);
166 }
167 if (( !MF->getSubtarget<AArch64Subtarget>().hardenSlsBlr() )) {
168 return fastEmitInst_r(MachineInstOpcode: AArch64::BLR, RC: &AArch64::GPR64RegClass, Op0);
169 }
170 return Register();
171}
172
173Register fastEmit_AArch64ISD_CALL_r(MVT VT, MVT RetVT, Register Op0) {
174 switch (VT.SimpleTy) {
175 case MVT::i64: return fastEmit_AArch64ISD_CALL_MVT_i64_r(RetVT, Op0);
176 default: return Register();
177 }
178}
179
180// FastEmit functions for AArch64ISD::COALESCER_BARRIER.
181
182Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_bf16_r(MVT RetVT, Register Op0) {
183 if (RetVT.SimpleTy != MVT::bf16)
184 return Register();
185 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR16, RC: &AArch64::FPR16RegClass, Op0);
186}
187
188Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f16_r(MVT RetVT, Register Op0) {
189 if (RetVT.SimpleTy != MVT::f16)
190 return Register();
191 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR16, RC: &AArch64::FPR16RegClass, Op0);
192}
193
194Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f32_r(MVT RetVT, Register Op0) {
195 if (RetVT.SimpleTy != MVT::f32)
196 return Register();
197 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR32, RC: &AArch64::FPR32RegClass, Op0);
198}
199
200Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f64_r(MVT RetVT, Register Op0) {
201 if (RetVT.SimpleTy != MVT::f64)
202 return Register();
203 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
204}
205
206Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f128_r(MVT RetVT, Register Op0) {
207 if (RetVT.SimpleTy != MVT::f128)
208 return Register();
209 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
210}
211
212Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i8_r(MVT RetVT, Register Op0) {
213 if (RetVT.SimpleTy != MVT::v8i8)
214 return Register();
215 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
216}
217
218Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v16i8_r(MVT RetVT, Register Op0) {
219 if (RetVT.SimpleTy != MVT::v16i8)
220 return Register();
221 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
222}
223
224Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i16_r(MVT RetVT, Register Op0) {
225 if (RetVT.SimpleTy != MVT::v4i16)
226 return Register();
227 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
228}
229
230Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i16_r(MVT RetVT, Register Op0) {
231 if (RetVT.SimpleTy != MVT::v8i16)
232 return Register();
233 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
234}
235
236Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i32_r(MVT RetVT, Register Op0) {
237 if (RetVT.SimpleTy != MVT::v2i32)
238 return Register();
239 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
240}
241
242Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i32_r(MVT RetVT, Register Op0) {
243 if (RetVT.SimpleTy != MVT::v4i32)
244 return Register();
245 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
246}
247
248Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1i64_r(MVT RetVT, Register Op0) {
249 if (RetVT.SimpleTy != MVT::v1i64)
250 return Register();
251 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
252}
253
254Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i64_r(MVT RetVT, Register Op0) {
255 if (RetVT.SimpleTy != MVT::v2i64)
256 return Register();
257 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
258}
259
260Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f16_r(MVT RetVT, Register Op0) {
261 if (RetVT.SimpleTy != MVT::v4f16)
262 return Register();
263 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
264}
265
266Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8f16_r(MVT RetVT, Register Op0) {
267 if (RetVT.SimpleTy != MVT::v8f16)
268 return Register();
269 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
270}
271
272Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4bf16_r(MVT RetVT, Register Op0) {
273 if (RetVT.SimpleTy != MVT::v4bf16)
274 return Register();
275 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
276}
277
278Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8bf16_r(MVT RetVT, Register Op0) {
279 if (RetVT.SimpleTy != MVT::v8bf16)
280 return Register();
281 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
282}
283
284Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f32_r(MVT RetVT, Register Op0) {
285 if (RetVT.SimpleTy != MVT::v2f32)
286 return Register();
287 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
288}
289
290Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f32_r(MVT RetVT, Register Op0) {
291 if (RetVT.SimpleTy != MVT::v4f32)
292 return Register();
293 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
294}
295
296Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1f64_r(MVT RetVT, Register Op0) {
297 if (RetVT.SimpleTy != MVT::v1f64)
298 return Register();
299 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
300}
301
302Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f64_r(MVT RetVT, Register Op0) {
303 if (RetVT.SimpleTy != MVT::v2f64)
304 return Register();
305 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
306}
307
308Register fastEmit_AArch64ISD_COALESCER_BARRIER_r(MVT VT, MVT RetVT, Register Op0) {
309 switch (VT.SimpleTy) {
310 case MVT::bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_bf16_r(RetVT, Op0);
311 case MVT::f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f16_r(RetVT, Op0);
312 case MVT::f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f32_r(RetVT, Op0);
313 case MVT::f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f64_r(RetVT, Op0);
314 case MVT::f128: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f128_r(RetVT, Op0);
315 case MVT::v8i8: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i8_r(RetVT, Op0);
316 case MVT::v16i8: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v16i8_r(RetVT, Op0);
317 case MVT::v4i16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i16_r(RetVT, Op0);
318 case MVT::v8i16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i16_r(RetVT, Op0);
319 case MVT::v2i32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i32_r(RetVT, Op0);
320 case MVT::v4i32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i32_r(RetVT, Op0);
321 case MVT::v1i64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1i64_r(RetVT, Op0);
322 case MVT::v2i64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i64_r(RetVT, Op0);
323 case MVT::v4f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f16_r(RetVT, Op0);
324 case MVT::v8f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8f16_r(RetVT, Op0);
325 case MVT::v4bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4bf16_r(RetVT, Op0);
326 case MVT::v8bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8bf16_r(RetVT, Op0);
327 case MVT::v2f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f32_r(RetVT, Op0);
328 case MVT::v4f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f32_r(RetVT, Op0);
329 case MVT::v1f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1f64_r(RetVT, Op0);
330 case MVT::v2f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f64_r(RetVT, Op0);
331 default: return Register();
332 }
333}
334
335// FastEmit functions for AArch64ISD::DUP.
336
337Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Register Op0) {
338 if ((Subtarget->isNeonAvailable())) {
339 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv8i8gpr, RC: &AArch64::FPR64RegClass, Op0);
340 }
341 return Register();
342}
343
344Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Register Op0) {
345 if ((Subtarget->isNeonAvailable())) {
346 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv16i8gpr, RC: &AArch64::FPR128RegClass, Op0);
347 }
348 return Register();
349}
350
351Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Register Op0) {
352 if ((Subtarget->isNeonAvailable())) {
353 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv4i16gpr, RC: &AArch64::FPR64RegClass, Op0);
354 }
355 return Register();
356}
357
358Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Register Op0) {
359 if ((Subtarget->isNeonAvailable())) {
360 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv8i16gpr, RC: &AArch64::FPR128RegClass, Op0);
361 }
362 return Register();
363}
364
365Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Register Op0) {
366 if ((Subtarget->isNeonAvailable())) {
367 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv2i32gpr, RC: &AArch64::FPR64RegClass, Op0);
368 }
369 return Register();
370}
371
372Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Register Op0) {
373 if ((Subtarget->isNeonAvailable())) {
374 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv4i32gpr, RC: &AArch64::FPR128RegClass, Op0);
375 }
376 return Register();
377}
378
379Register fastEmit_AArch64ISD_DUP_MVT_i32_r(MVT RetVT, Register Op0) {
380switch (RetVT.SimpleTy) {
381 case MVT::v8i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Op0);
382 case MVT::v16i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Op0);
383 case MVT::v4i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Op0);
384 case MVT::v8i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Op0);
385 case MVT::v2i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Op0);
386 case MVT::v4i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Op0);
387 default: return Register();
388}
389}
390
391Register fastEmit_AArch64ISD_DUP_MVT_i64_r(MVT RetVT, Register Op0) {
392 if (RetVT.SimpleTy != MVT::v2i64)
393 return Register();
394 if ((Subtarget->isNeonAvailable())) {
395 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv2i64gpr, RC: &AArch64::FPR128RegClass, Op0);
396 }
397 return Register();
398}
399
400Register fastEmit_AArch64ISD_DUP_r(MVT VT, MVT RetVT, Register Op0) {
401 switch (VT.SimpleTy) {
402 case MVT::i32: return fastEmit_AArch64ISD_DUP_MVT_i32_r(RetVT, Op0);
403 case MVT::i64: return fastEmit_AArch64ISD_DUP_MVT_i64_r(RetVT, Op0);
404 default: return Register();
405 }
406}
407
408// FastEmit functions for AArch64ISD::FCVTXN.
409
410Register fastEmit_AArch64ISD_FCVTXN_MVT_f64_r(MVT RetVT, Register Op0) {
411 if (RetVT.SimpleTy != MVT::f32)
412 return Register();
413 if ((Subtarget->isNeonAvailable())) {
414 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTXNv1i64, RC: &AArch64::FPR32RegClass, Op0);
415 }
416 return Register();
417}
418
419Register fastEmit_AArch64ISD_FCVTXN_MVT_v2f64_r(MVT RetVT, Register Op0) {
420 if (RetVT.SimpleTy != MVT::v2f32)
421 return Register();
422 if ((Subtarget->isNeonAvailable())) {
423 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTXNv2f32, RC: &AArch64::FPR64RegClass, Op0);
424 }
425 return Register();
426}
427
428Register fastEmit_AArch64ISD_FCVTXN_r(MVT VT, MVT RetVT, Register Op0) {
429 switch (VT.SimpleTy) {
430 case MVT::f64: return fastEmit_AArch64ISD_FCVTXN_MVT_f64_r(RetVT, Op0);
431 case MVT::v2f64: return fastEmit_AArch64ISD_FCVTXN_MVT_v2f64_r(RetVT, Op0);
432 default: return Register();
433 }
434}
435
436// FastEmit functions for AArch64ISD::FRECPE.
437
438Register fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(MVT RetVT, Register Op0) {
439 if (RetVT.SimpleTy != MVT::v2f32)
440 return Register();
441 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv2f32, RC: &AArch64::FPR64RegClass, Op0);
442}
443
444Register fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(MVT RetVT, Register Op0) {
445 if (RetVT.SimpleTy != MVT::v4f32)
446 return Register();
447 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv4f32, RC: &AArch64::FPR128RegClass, Op0);
448}
449
450Register fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(MVT RetVT, Register Op0) {
451 if (RetVT.SimpleTy != MVT::v2f64)
452 return Register();
453 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv2f64, RC: &AArch64::FPR128RegClass, Op0);
454}
455
456Register fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
457 if (RetVT.SimpleTy != MVT::nxv8f16)
458 return Register();
459 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
460 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
461 }
462 return Register();
463}
464
465Register fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
466 if (RetVT.SimpleTy != MVT::nxv4f32)
467 return Register();
468 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
469 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
470 }
471 return Register();
472}
473
474Register fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
475 if (RetVT.SimpleTy != MVT::nxv2f64)
476 return Register();
477 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
478 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
479 }
480 return Register();
481}
482
483Register fastEmit_AArch64ISD_FRECPE_r(MVT VT, MVT RetVT, Register Op0) {
484 switch (VT.SimpleTy) {
485 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(RetVT, Op0);
486 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(RetVT, Op0);
487 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(RetVT, Op0);
488 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(RetVT, Op0);
489 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(RetVT, Op0);
490 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(RetVT, Op0);
491 default: return Register();
492 }
493}
494
495// FastEmit functions for AArch64ISD::FRSQRTE.
496
497Register fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(MVT RetVT, Register Op0) {
498 if (RetVT.SimpleTy != MVT::v2f32)
499 return Register();
500 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv2f32, RC: &AArch64::FPR64RegClass, Op0);
501}
502
503Register fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(MVT RetVT, Register Op0) {
504 if (RetVT.SimpleTy != MVT::v4f32)
505 return Register();
506 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv4f32, RC: &AArch64::FPR128RegClass, Op0);
507}
508
509Register fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(MVT RetVT, Register Op0) {
510 if (RetVT.SimpleTy != MVT::v2f64)
511 return Register();
512 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv2f64, RC: &AArch64::FPR128RegClass, Op0);
513}
514
515Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
516 if (RetVT.SimpleTy != MVT::nxv8f16)
517 return Register();
518 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
519 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
520 }
521 return Register();
522}
523
524Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
525 if (RetVT.SimpleTy != MVT::nxv4f32)
526 return Register();
527 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
528 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
529 }
530 return Register();
531}
532
533Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
534 if (RetVT.SimpleTy != MVT::nxv2f64)
535 return Register();
536 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
537 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
538 }
539 return Register();
540}
541
542Register fastEmit_AArch64ISD_FRSQRTE_r(MVT VT, MVT RetVT, Register Op0) {
543 switch (VT.SimpleTy) {
544 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(RetVT, Op0);
545 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(RetVT, Op0);
546 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(RetVT, Op0);
547 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(RetVT, Op0);
548 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(RetVT, Op0);
549 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(RetVT, Op0);
550 default: return Register();
551 }
552}
553
554// FastEmit functions for AArch64ISD::PROBED_ALLOCA.
555
556Register fastEmit_AArch64ISD_PROBED_ALLOCA_MVT_i64_r(MVT RetVT, Register Op0) {
557 if (RetVT.SimpleTy != MVT::isVoid)
558 return Register();
559 return fastEmitInst_r(MachineInstOpcode: AArch64::PROBED_STACKALLOC_DYN, RC: &AArch64::GPR64commonRegClass, Op0);
560}
561
562Register fastEmit_AArch64ISD_PROBED_ALLOCA_r(MVT VT, MVT RetVT, Register Op0) {
563 switch (VT.SimpleTy) {
564 case MVT::i64: return fastEmit_AArch64ISD_PROBED_ALLOCA_MVT_i64_r(RetVT, Op0);
565 default: return Register();
566 }
567}
568
569// FastEmit functions for AArch64ISD::REV16.
570
571Register fastEmit_AArch64ISD_REV16_MVT_i32_r(MVT RetVT, Register Op0) {
572 if (RetVT.SimpleTy != MVT::i32)
573 return Register();
574 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16Wr, RC: &AArch64::GPR32RegClass, Op0);
575}
576
577Register fastEmit_AArch64ISD_REV16_MVT_i64_r(MVT RetVT, Register Op0) {
578 if (RetVT.SimpleTy != MVT::i64)
579 return Register();
580 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16Xr, RC: &AArch64::GPR64RegClass, Op0);
581}
582
583Register fastEmit_AArch64ISD_REV16_r(MVT VT, MVT RetVT, Register Op0) {
584 switch (VT.SimpleTy) {
585 case MVT::i32: return fastEmit_AArch64ISD_REV16_MVT_i32_r(RetVT, Op0);
586 case MVT::i64: return fastEmit_AArch64ISD_REV16_MVT_i64_r(RetVT, Op0);
587 default: return Register();
588 }
589}
590
591// FastEmit functions for AArch64ISD::REV32.
592
593Register fastEmit_AArch64ISD_REV32_MVT_v8i8_r(MVT RetVT, Register Op0) {
594 if (RetVT.SimpleTy != MVT::v8i8)
595 return Register();
596 if ((Subtarget->isNeonAvailable())) {
597 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
598 }
599 return Register();
600}
601
602Register fastEmit_AArch64ISD_REV32_MVT_v16i8_r(MVT RetVT, Register Op0) {
603 if (RetVT.SimpleTy != MVT::v16i8)
604 return Register();
605 if ((Subtarget->isNeonAvailable())) {
606 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
607 }
608 return Register();
609}
610
611Register fastEmit_AArch64ISD_REV32_MVT_v4i16_r(MVT RetVT, Register Op0) {
612 if (RetVT.SimpleTy != MVT::v4i16)
613 return Register();
614 if ((Subtarget->isNeonAvailable())) {
615 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
616 }
617 return Register();
618}
619
620Register fastEmit_AArch64ISD_REV32_MVT_v8i16_r(MVT RetVT, Register Op0) {
621 if (RetVT.SimpleTy != MVT::v8i16)
622 return Register();
623 if ((Subtarget->isNeonAvailable())) {
624 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
625 }
626 return Register();
627}
628
629Register fastEmit_AArch64ISD_REV32_MVT_v4f16_r(MVT RetVT, Register Op0) {
630 if (RetVT.SimpleTy != MVT::v4f16)
631 return Register();
632 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
633}
634
635Register fastEmit_AArch64ISD_REV32_MVT_v8f16_r(MVT RetVT, Register Op0) {
636 if (RetVT.SimpleTy != MVT::v8f16)
637 return Register();
638 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
639}
640
641Register fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(MVT RetVT, Register Op0) {
642 if (RetVT.SimpleTy != MVT::v4bf16)
643 return Register();
644 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
645}
646
647Register fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(MVT RetVT, Register Op0) {
648 if (RetVT.SimpleTy != MVT::v8bf16)
649 return Register();
650 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
651}
652
653Register fastEmit_AArch64ISD_REV32_r(MVT VT, MVT RetVT, Register Op0) {
654 switch (VT.SimpleTy) {
655 case MVT::v8i8: return fastEmit_AArch64ISD_REV32_MVT_v8i8_r(RetVT, Op0);
656 case MVT::v16i8: return fastEmit_AArch64ISD_REV32_MVT_v16i8_r(RetVT, Op0);
657 case MVT::v4i16: return fastEmit_AArch64ISD_REV32_MVT_v4i16_r(RetVT, Op0);
658 case MVT::v8i16: return fastEmit_AArch64ISD_REV32_MVT_v8i16_r(RetVT, Op0);
659 case MVT::v4f16: return fastEmit_AArch64ISD_REV32_MVT_v4f16_r(RetVT, Op0);
660 case MVT::v8f16: return fastEmit_AArch64ISD_REV32_MVT_v8f16_r(RetVT, Op0);
661 case MVT::v4bf16: return fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(RetVT, Op0);
662 case MVT::v8bf16: return fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(RetVT, Op0);
663 default: return Register();
664 }
665}
666
667// FastEmit functions for AArch64ISD::REV64.
668
669Register fastEmit_AArch64ISD_REV64_MVT_v8i8_r(MVT RetVT, Register Op0) {
670 if (RetVT.SimpleTy != MVT::v8i8)
671 return Register();
672 if ((Subtarget->isNeonAvailable())) {
673 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
674 }
675 return Register();
676}
677
678Register fastEmit_AArch64ISD_REV64_MVT_v16i8_r(MVT RetVT, Register Op0) {
679 if (RetVT.SimpleTy != MVT::v16i8)
680 return Register();
681 if ((Subtarget->isNeonAvailable())) {
682 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
683 }
684 return Register();
685}
686
687Register fastEmit_AArch64ISD_REV64_MVT_v4i16_r(MVT RetVT, Register Op0) {
688 if (RetVT.SimpleTy != MVT::v4i16)
689 return Register();
690 if ((Subtarget->isNeonAvailable())) {
691 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
692 }
693 return Register();
694}
695
696Register fastEmit_AArch64ISD_REV64_MVT_v8i16_r(MVT RetVT, Register Op0) {
697 if (RetVT.SimpleTy != MVT::v8i16)
698 return Register();
699 if ((Subtarget->isNeonAvailable())) {
700 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
701 }
702 return Register();
703}
704
705Register fastEmit_AArch64ISD_REV64_MVT_v2i32_r(MVT RetVT, Register Op0) {
706 if (RetVT.SimpleTy != MVT::v2i32)
707 return Register();
708 if ((Subtarget->isNeonAvailable())) {
709 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
710 }
711 return Register();
712}
713
714Register fastEmit_AArch64ISD_REV64_MVT_v4i32_r(MVT RetVT, Register Op0) {
715 if (RetVT.SimpleTy != MVT::v4i32)
716 return Register();
717 if ((Subtarget->isNeonAvailable())) {
718 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
719 }
720 return Register();
721}
722
723Register fastEmit_AArch64ISD_REV64_MVT_v4f16_r(MVT RetVT, Register Op0) {
724 if (RetVT.SimpleTy != MVT::v4f16)
725 return Register();
726 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
727}
728
729Register fastEmit_AArch64ISD_REV64_MVT_v8f16_r(MVT RetVT, Register Op0) {
730 if (RetVT.SimpleTy != MVT::v8f16)
731 return Register();
732 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
733}
734
735Register fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(MVT RetVT, Register Op0) {
736 if (RetVT.SimpleTy != MVT::v4bf16)
737 return Register();
738 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
739}
740
741Register fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(MVT RetVT, Register Op0) {
742 if (RetVT.SimpleTy != MVT::v8bf16)
743 return Register();
744 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
745}
746
747Register fastEmit_AArch64ISD_REV64_MVT_v2f32_r(MVT RetVT, Register Op0) {
748 if (RetVT.SimpleTy != MVT::v2f32)
749 return Register();
750 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
751}
752
753Register fastEmit_AArch64ISD_REV64_MVT_v4f32_r(MVT RetVT, Register Op0) {
754 if (RetVT.SimpleTy != MVT::v4f32)
755 return Register();
756 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
757}
758
759Register fastEmit_AArch64ISD_REV64_r(MVT VT, MVT RetVT, Register Op0) {
760 switch (VT.SimpleTy) {
761 case MVT::v8i8: return fastEmit_AArch64ISD_REV64_MVT_v8i8_r(RetVT, Op0);
762 case MVT::v16i8: return fastEmit_AArch64ISD_REV64_MVT_v16i8_r(RetVT, Op0);
763 case MVT::v4i16: return fastEmit_AArch64ISD_REV64_MVT_v4i16_r(RetVT, Op0);
764 case MVT::v8i16: return fastEmit_AArch64ISD_REV64_MVT_v8i16_r(RetVT, Op0);
765 case MVT::v2i32: return fastEmit_AArch64ISD_REV64_MVT_v2i32_r(RetVT, Op0);
766 case MVT::v4i32: return fastEmit_AArch64ISD_REV64_MVT_v4i32_r(RetVT, Op0);
767 case MVT::v4f16: return fastEmit_AArch64ISD_REV64_MVT_v4f16_r(RetVT, Op0);
768 case MVT::v8f16: return fastEmit_AArch64ISD_REV64_MVT_v8f16_r(RetVT, Op0);
769 case MVT::v4bf16: return fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(RetVT, Op0);
770 case MVT::v8bf16: return fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(RetVT, Op0);
771 case MVT::v2f32: return fastEmit_AArch64ISD_REV64_MVT_v2f32_r(RetVT, Op0);
772 case MVT::v4f32: return fastEmit_AArch64ISD_REV64_MVT_v4f32_r(RetVT, Op0);
773 default: return Register();
774 }
775}
776
777// FastEmit functions for AArch64ISD::SADDLP.
778
779Register fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(MVT RetVT, Register Op0) {
780 if (RetVT.SimpleTy != MVT::v4i16)
781 return Register();
782 if ((Subtarget->isNeonAvailable())) {
783 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv8i8_v4i16, RC: &AArch64::FPR64RegClass, Op0);
784 }
785 return Register();
786}
787
788Register fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(MVT RetVT, Register Op0) {
789 if (RetVT.SimpleTy != MVT::v8i16)
790 return Register();
791 if ((Subtarget->isNeonAvailable())) {
792 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv16i8_v8i16, RC: &AArch64::FPR128RegClass, Op0);
793 }
794 return Register();
795}
796
797Register fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(MVT RetVT, Register Op0) {
798 if (RetVT.SimpleTy != MVT::v2i32)
799 return Register();
800 if ((Subtarget->isNeonAvailable())) {
801 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv4i16_v2i32, RC: &AArch64::FPR64RegClass, Op0);
802 }
803 return Register();
804}
805
806Register fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(MVT RetVT, Register Op0) {
807 if (RetVT.SimpleTy != MVT::v4i32)
808 return Register();
809 if ((Subtarget->isNeonAvailable())) {
810 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv8i16_v4i32, RC: &AArch64::FPR128RegClass, Op0);
811 }
812 return Register();
813}
814
815Register fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(MVT RetVT, Register Op0) {
816 if (RetVT.SimpleTy != MVT::v1i64)
817 return Register();
818 if ((Subtarget->isNeonAvailable())) {
819 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv2i32_v1i64, RC: &AArch64::FPR64RegClass, Op0);
820 }
821 return Register();
822}
823
824Register fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(MVT RetVT, Register Op0) {
825 if (RetVT.SimpleTy != MVT::v2i64)
826 return Register();
827 if ((Subtarget->isNeonAvailable())) {
828 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv4i32_v2i64, RC: &AArch64::FPR128RegClass, Op0);
829 }
830 return Register();
831}
832
833Register fastEmit_AArch64ISD_SADDLP_r(MVT VT, MVT RetVT, Register Op0) {
834 switch (VT.SimpleTy) {
835 case MVT::v8i8: return fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(RetVT, Op0);
836 case MVT::v16i8: return fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(RetVT, Op0);
837 case MVT::v4i16: return fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(RetVT, Op0);
838 case MVT::v8i16: return fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(RetVT, Op0);
839 case MVT::v2i32: return fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(RetVT, Op0);
840 case MVT::v4i32: return fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(RetVT, Op0);
841 default: return Register();
842 }
843}
844
845// FastEmit functions for AArch64ISD::SITOF.
846
847Register fastEmit_AArch64ISD_SITOF_MVT_f16_r(MVT RetVT, Register Op0) {
848 if (RetVT.SimpleTy != MVT::f16)
849 return Register();
850 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
851 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i16, RC: &AArch64::FPR16RegClass, Op0);
852 }
853 return Register();
854}
855
856Register fastEmit_AArch64ISD_SITOF_MVT_f32_r(MVT RetVT, Register Op0) {
857 if (RetVT.SimpleTy != MVT::f32)
858 return Register();
859 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
860 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i32, RC: &AArch64::FPR32RegClass, Op0);
861 }
862 return Register();
863}
864
865Register fastEmit_AArch64ISD_SITOF_MVT_f64_r(MVT RetVT, Register Op0) {
866 if (RetVT.SimpleTy != MVT::f64)
867 return Register();
868 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
869 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i64, RC: &AArch64::FPR64RegClass, Op0);
870 }
871 return Register();
872}
873
874Register fastEmit_AArch64ISD_SITOF_r(MVT VT, MVT RetVT, Register Op0) {
875 switch (VT.SimpleTy) {
876 case MVT::f16: return fastEmit_AArch64ISD_SITOF_MVT_f16_r(RetVT, Op0);
877 case MVT::f32: return fastEmit_AArch64ISD_SITOF_MVT_f32_r(RetVT, Op0);
878 case MVT::f64: return fastEmit_AArch64ISD_SITOF_MVT_f64_r(RetVT, Op0);
879 default: return Register();
880 }
881}
882
883// FastEmit functions for AArch64ISD::SQABS.
884
885Register fastEmit_AArch64ISD_SQABS_MVT_f32_r(MVT RetVT, Register Op0) {
886 if (RetVT.SimpleTy != MVT::f32)
887 return Register();
888 if ((Subtarget->isNeonAvailable())) {
889 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv1i32, RC: &AArch64::FPR32RegClass, Op0);
890 }
891 return Register();
892}
893
894Register fastEmit_AArch64ISD_SQABS_MVT_f64_r(MVT RetVT, Register Op0) {
895 if (RetVT.SimpleTy != MVT::f64)
896 return Register();
897 if ((Subtarget->isNeonAvailable())) {
898 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
899 }
900 return Register();
901}
902
903Register fastEmit_AArch64ISD_SQABS_r(MVT VT, MVT RetVT, Register Op0) {
904 switch (VT.SimpleTy) {
905 case MVT::f32: return fastEmit_AArch64ISD_SQABS_MVT_f32_r(RetVT, Op0);
906 case MVT::f64: return fastEmit_AArch64ISD_SQABS_MVT_f64_r(RetVT, Op0);
907 default: return Register();
908 }
909}
910
911// FastEmit functions for AArch64ISD::SQNEG.
912
913Register fastEmit_AArch64ISD_SQNEG_MVT_f32_r(MVT RetVT, Register Op0) {
914 if (RetVT.SimpleTy != MVT::f32)
915 return Register();
916 if ((Subtarget->isNeonAvailable())) {
917 return fastEmitInst_r(MachineInstOpcode: AArch64::SQNEGv1i32, RC: &AArch64::FPR32RegClass, Op0);
918 }
919 return Register();
920}
921
922Register fastEmit_AArch64ISD_SQNEG_MVT_f64_r(MVT RetVT, Register Op0) {
923 if (RetVT.SimpleTy != MVT::f64)
924 return Register();
925 if ((Subtarget->isNeonAvailable())) {
926 return fastEmitInst_r(MachineInstOpcode: AArch64::SQNEGv1i64, RC: &AArch64::FPR64RegClass, Op0);
927 }
928 return Register();
929}
930
931Register fastEmit_AArch64ISD_SQNEG_r(MVT VT, MVT RetVT, Register Op0) {
932 switch (VT.SimpleTy) {
933 case MVT::f32: return fastEmit_AArch64ISD_SQNEG_MVT_f32_r(RetVT, Op0);
934 case MVT::f64: return fastEmit_AArch64ISD_SQNEG_MVT_f64_r(RetVT, Op0);
935 default: return Register();
936 }
937}
938
939// FastEmit functions for AArch64ISD::SUNPKHI.
940
941Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
942 if (RetVT.SimpleTy != MVT::nxv8i16)
943 return Register();
944 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
945 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
946 }
947 return Register();
948}
949
950Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
951 if (RetVT.SimpleTy != MVT::nxv4i32)
952 return Register();
953 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
954 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
955 }
956 return Register();
957}
958
959Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
960 if (RetVT.SimpleTy != MVT::nxv2i64)
961 return Register();
962 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
963 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
964 }
965 return Register();
966}
967
968Register fastEmit_AArch64ISD_SUNPKHI_r(MVT VT, MVT RetVT, Register Op0) {
969 switch (VT.SimpleTy) {
970 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(RetVT, Op0);
971 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(RetVT, Op0);
972 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(RetVT, Op0);
973 default: return Register();
974 }
975}
976
977// FastEmit functions for AArch64ISD::SUNPKLO.
978
979Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
980 if (RetVT.SimpleTy != MVT::nxv8i16)
981 return Register();
982 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
983 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
984 }
985 return Register();
986}
987
988Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
989 if (RetVT.SimpleTy != MVT::nxv4i32)
990 return Register();
991 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
992 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
993 }
994 return Register();
995}
996
997Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
998 if (RetVT.SimpleTy != MVT::nxv2i64)
999 return Register();
1000 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1001 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1002 }
1003 return Register();
1004}
1005
1006Register fastEmit_AArch64ISD_SUNPKLO_r(MVT VT, MVT RetVT, Register Op0) {
1007 switch (VT.SimpleTy) {
1008 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(RetVT, Op0);
1009 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(RetVT, Op0);
1010 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(RetVT, Op0);
1011 default: return Register();
1012 }
1013}
1014
1015// FastEmit functions for AArch64ISD::UADDLP.
1016
1017Register fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(MVT RetVT, Register Op0) {
1018 if (RetVT.SimpleTy != MVT::v4i16)
1019 return Register();
1020 if ((Subtarget->isNeonAvailable())) {
1021 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv8i8_v4i16, RC: &AArch64::FPR64RegClass, Op0);
1022 }
1023 return Register();
1024}
1025
1026Register fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(MVT RetVT, Register Op0) {
1027 if (RetVT.SimpleTy != MVT::v8i16)
1028 return Register();
1029 if ((Subtarget->isNeonAvailable())) {
1030 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv16i8_v8i16, RC: &AArch64::FPR128RegClass, Op0);
1031 }
1032 return Register();
1033}
1034
1035Register fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(MVT RetVT, Register Op0) {
1036 if (RetVT.SimpleTy != MVT::v2i32)
1037 return Register();
1038 if ((Subtarget->isNeonAvailable())) {
1039 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv4i16_v2i32, RC: &AArch64::FPR64RegClass, Op0);
1040 }
1041 return Register();
1042}
1043
1044Register fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(MVT RetVT, Register Op0) {
1045 if (RetVT.SimpleTy != MVT::v4i32)
1046 return Register();
1047 if ((Subtarget->isNeonAvailable())) {
1048 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv8i16_v4i32, RC: &AArch64::FPR128RegClass, Op0);
1049 }
1050 return Register();
1051}
1052
1053Register fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(MVT RetVT, Register Op0) {
1054 if (RetVT.SimpleTy != MVT::v1i64)
1055 return Register();
1056 if ((Subtarget->isNeonAvailable())) {
1057 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv2i32_v1i64, RC: &AArch64::FPR64RegClass, Op0);
1058 }
1059 return Register();
1060}
1061
1062Register fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(MVT RetVT, Register Op0) {
1063 if (RetVT.SimpleTy != MVT::v2i64)
1064 return Register();
1065 if ((Subtarget->isNeonAvailable())) {
1066 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv4i32_v2i64, RC: &AArch64::FPR128RegClass, Op0);
1067 }
1068 return Register();
1069}
1070
1071Register fastEmit_AArch64ISD_UADDLP_r(MVT VT, MVT RetVT, Register Op0) {
1072 switch (VT.SimpleTy) {
1073 case MVT::v8i8: return fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(RetVT, Op0);
1074 case MVT::v16i8: return fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(RetVT, Op0);
1075 case MVT::v4i16: return fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(RetVT, Op0);
1076 case MVT::v8i16: return fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(RetVT, Op0);
1077 case MVT::v2i32: return fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(RetVT, Op0);
1078 case MVT::v4i32: return fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(RetVT, Op0);
1079 default: return Register();
1080 }
1081}
1082
1083// FastEmit functions for AArch64ISD::UITOF.
1084
1085Register fastEmit_AArch64ISD_UITOF_MVT_f16_r(MVT RetVT, Register Op0) {
1086 if (RetVT.SimpleTy != MVT::f16)
1087 return Register();
1088 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
1089 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i16, RC: &AArch64::FPR16RegClass, Op0);
1090 }
1091 return Register();
1092}
1093
1094Register fastEmit_AArch64ISD_UITOF_MVT_f32_r(MVT RetVT, Register Op0) {
1095 if (RetVT.SimpleTy != MVT::f32)
1096 return Register();
1097 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
1098 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i32, RC: &AArch64::FPR32RegClass, Op0);
1099 }
1100 return Register();
1101}
1102
1103Register fastEmit_AArch64ISD_UITOF_MVT_f64_r(MVT RetVT, Register Op0) {
1104 if (RetVT.SimpleTy != MVT::f64)
1105 return Register();
1106 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
1107 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i64, RC: &AArch64::FPR64RegClass, Op0);
1108 }
1109 return Register();
1110}
1111
1112Register fastEmit_AArch64ISD_UITOF_r(MVT VT, MVT RetVT, Register Op0) {
1113 switch (VT.SimpleTy) {
1114 case MVT::f16: return fastEmit_AArch64ISD_UITOF_MVT_f16_r(RetVT, Op0);
1115 case MVT::f32: return fastEmit_AArch64ISD_UITOF_MVT_f32_r(RetVT, Op0);
1116 case MVT::f64: return fastEmit_AArch64ISD_UITOF_MVT_f64_r(RetVT, Op0);
1117 default: return Register();
1118 }
1119}
1120
1121// FastEmit functions for AArch64ISD::UUNPKHI.
1122
1123Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
1124 if (RetVT.SimpleTy != MVT::nxv8i16)
1125 return Register();
1126 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1127 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1128 }
1129 return Register();
1130}
1131
1132Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
1133 if (RetVT.SimpleTy != MVT::nxv4i32)
1134 return Register();
1135 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1136 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1137 }
1138 return Register();
1139}
1140
1141Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
1142 if (RetVT.SimpleTy != MVT::nxv2i64)
1143 return Register();
1144 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1145 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1146 }
1147 return Register();
1148}
1149
1150Register fastEmit_AArch64ISD_UUNPKHI_r(MVT VT, MVT RetVT, Register Op0) {
1151 switch (VT.SimpleTy) {
1152 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(RetVT, Op0);
1153 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(RetVT, Op0);
1154 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(RetVT, Op0);
1155 default: return Register();
1156 }
1157}
1158
1159// FastEmit functions for AArch64ISD::UUNPKLO.
1160
1161Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
1162 if (RetVT.SimpleTy != MVT::nxv8i16)
1163 return Register();
1164 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1165 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1166 }
1167 return Register();
1168}
1169
1170Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
1171 if (RetVT.SimpleTy != MVT::nxv4i32)
1172 return Register();
1173 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1174 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1175 }
1176 return Register();
1177}
1178
1179Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
1180 if (RetVT.SimpleTy != MVT::nxv2i64)
1181 return Register();
1182 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1183 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1184 }
1185 return Register();
1186}
1187
1188Register fastEmit_AArch64ISD_UUNPKLO_r(MVT VT, MVT RetVT, Register Op0) {
1189 switch (VT.SimpleTy) {
1190 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(RetVT, Op0);
1191 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(RetVT, Op0);
1192 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(RetVT, Op0);
1193 default: return Register();
1194 }
1195}
1196
1197// FastEmit functions for ISD::ABS.
1198
1199Register fastEmit_ISD_ABS_MVT_i32_r(MVT RetVT, Register Op0) {
1200 if (RetVT.SimpleTy != MVT::i32)
1201 return Register();
1202 if ((Subtarget->hasCSSC())) {
1203 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSWr, RC: &AArch64::GPR32RegClass, Op0);
1204 }
1205 return Register();
1206}
1207
1208Register fastEmit_ISD_ABS_MVT_i64_r(MVT RetVT, Register Op0) {
1209 if (RetVT.SimpleTy != MVT::i64)
1210 return Register();
1211 if ((!Subtarget->hasCSSC())) {
1212 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
1213 }
1214 if ((Subtarget->hasCSSC())) {
1215 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSXr, RC: &AArch64::GPR64RegClass, Op0);
1216 }
1217 return Register();
1218}
1219
1220Register fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, Register Op0) {
1221 if (RetVT.SimpleTy != MVT::v8i8)
1222 return Register();
1223 if ((Subtarget->isNeonAvailable())) {
1224 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv8i8, RC: &AArch64::FPR64RegClass, Op0);
1225 }
1226 return Register();
1227}
1228
1229Register fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, Register Op0) {
1230 if (RetVT.SimpleTy != MVT::v16i8)
1231 return Register();
1232 if ((Subtarget->isNeonAvailable())) {
1233 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv16i8, RC: &AArch64::FPR128RegClass, Op0);
1234 }
1235 return Register();
1236}
1237
1238Register fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, Register Op0) {
1239 if (RetVT.SimpleTy != MVT::v4i16)
1240 return Register();
1241 if ((Subtarget->isNeonAvailable())) {
1242 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv4i16, RC: &AArch64::FPR64RegClass, Op0);
1243 }
1244 return Register();
1245}
1246
1247Register fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, Register Op0) {
1248 if (RetVT.SimpleTy != MVT::v8i16)
1249 return Register();
1250 if ((Subtarget->isNeonAvailable())) {
1251 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv8i16, RC: &AArch64::FPR128RegClass, Op0);
1252 }
1253 return Register();
1254}
1255
1256Register fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, Register Op0) {
1257 if (RetVT.SimpleTy != MVT::v2i32)
1258 return Register();
1259 if ((Subtarget->isNeonAvailable())) {
1260 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv2i32, RC: &AArch64::FPR64RegClass, Op0);
1261 }
1262 return Register();
1263}
1264
1265Register fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, Register Op0) {
1266 if (RetVT.SimpleTy != MVT::v4i32)
1267 return Register();
1268 if ((Subtarget->isNeonAvailable())) {
1269 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv4i32, RC: &AArch64::FPR128RegClass, Op0);
1270 }
1271 return Register();
1272}
1273
1274Register fastEmit_ISD_ABS_MVT_v1i64_r(MVT RetVT, Register Op0) {
1275 if (RetVT.SimpleTy != MVT::v1i64)
1276 return Register();
1277 if ((Subtarget->isNeonAvailable())) {
1278 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
1279 }
1280 return Register();
1281}
1282
1283Register fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, Register Op0) {
1284 if (RetVT.SimpleTy != MVT::v2i64)
1285 return Register();
1286 if ((Subtarget->isNeonAvailable())) {
1287 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv2i64, RC: &AArch64::FPR128RegClass, Op0);
1288 }
1289 return Register();
1290}
1291
1292Register fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, Register Op0) {
1293 switch (VT.SimpleTy) {
1294 case MVT::i32: return fastEmit_ISD_ABS_MVT_i32_r(RetVT, Op0);
1295 case MVT::i64: return fastEmit_ISD_ABS_MVT_i64_r(RetVT, Op0);
1296 case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0);
1297 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
1298 case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0);
1299 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
1300 case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0);
1301 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
1302 case MVT::v1i64: return fastEmit_ISD_ABS_MVT_v1i64_r(RetVT, Op0);
1303 case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0);
1304 default: return Register();
1305 }
1306}
1307
1308// FastEmit functions for ISD::BITCAST.
1309
1310Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Register Op0) {
1311 if ((!Subtarget->isLittleEndian())) {
1312 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1313 }
1314 return Register();
1315}
1316
1317Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Register Op0) {
1318 if ((!Subtarget->isLittleEndian())) {
1319 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1320 }
1321 return Register();
1322}
1323
1324Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Register Op0) {
1325 if ((!Subtarget->isLittleEndian())) {
1326 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1327 }
1328 return Register();
1329}
1330
1331Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Register Op0) {
1332 if ((!Subtarget->isLittleEndian())) {
1333 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1334 }
1335 return Register();
1336}
1337
1338Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Register Op0) {
1339 if ((!Subtarget->isLittleEndian())) {
1340 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1341 }
1342 return Register();
1343}
1344
1345Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Register Op0) {
1346 if ((!Subtarget->isLittleEndian())) {
1347 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1348 }
1349 return Register();
1350}
1351
1352Register fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, Register Op0) {
1353switch (RetVT.SimpleTy) {
1354 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0);
1355 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0);
1356 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0);
1357 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0);
1358 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Op0);
1359 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0);
1360 default: return Register();
1361}
1362}
1363
1364Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Register Op0) {
1365 if ((!Subtarget->isLittleEndian())) {
1366 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1367 }
1368 return Register();
1369}
1370
1371Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Register Op0) {
1372 if ((!Subtarget->isLittleEndian())) {
1373 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1374 }
1375 return Register();
1376}
1377
1378Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Register Op0) {
1379 if ((!Subtarget->isLittleEndian())) {
1380 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1381 }
1382 return Register();
1383}
1384
1385Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Register Op0) {
1386 if ((!Subtarget->isLittleEndian())) {
1387 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1388 }
1389 return Register();
1390}
1391
1392Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Register Op0) {
1393 if ((!Subtarget->isLittleEndian())) {
1394 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1395 }
1396 return Register();
1397}
1398
1399Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Register Op0) {
1400 if ((!Subtarget->isLittleEndian())) {
1401 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1402 }
1403 return Register();
1404}
1405
1406Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Register Op0) {
1407 if ((!Subtarget->isLittleEndian())) {
1408 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1409 }
1410 return Register();
1411}
1412
1413Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Register Op0) {
1414 if ((!Subtarget->isLittleEndian())) {
1415 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1416 }
1417 return Register();
1418}
1419
1420Register fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, Register Op0) {
1421switch (RetVT.SimpleTy) {
1422 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0);
1423 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0);
1424 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0);
1425 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0);
1426 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0);
1427 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Op0);
1428 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0);
1429 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Op0);
1430 default: return Register();
1431}
1432}
1433
1434Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Register Op0) {
1435 if ((!Subtarget->isLittleEndian())) {
1436 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1437 }
1438 return Register();
1439}
1440
1441Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Register Op0) {
1442 if ((!Subtarget->isLittleEndian())) {
1443 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1444 }
1445 return Register();
1446}
1447
1448Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Register Op0) {
1449 if ((!Subtarget->isLittleEndian())) {
1450 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1451 }
1452 return Register();
1453}
1454
1455Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Register Op0) {
1456 if ((!Subtarget->isLittleEndian())) {
1457 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1458 }
1459 return Register();
1460}
1461
1462Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Register Op0) {
1463 if ((!Subtarget->isLittleEndian())) {
1464 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1465 }
1466 return Register();
1467}
1468
1469Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Register Op0) {
1470 if ((!Subtarget->isLittleEndian())) {
1471 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1472 }
1473 return Register();
1474}
1475
1476Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Register Op0) {
1477 if ((!Subtarget->isLittleEndian())) {
1478 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1479 }
1480 return Register();
1481}
1482
1483Register fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, Register Op0) {
1484switch (RetVT.SimpleTy) {
1485 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0);
1486 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0);
1487 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0);
1488 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0);
1489 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Op0);
1490 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0);
1491 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0);
1492 default: return Register();
1493}
1494}
1495
1496Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Register Op0) {
1497 if ((!Subtarget->isLittleEndian())) {
1498 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1499 }
1500 return Register();
1501}
1502
1503Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Register Op0) {
1504 if ((!Subtarget->isLittleEndian())) {
1505 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1506 }
1507 return Register();
1508}
1509
1510Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Register Op0) {
1511 if ((!Subtarget->isLittleEndian())) {
1512 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1513 }
1514 return Register();
1515}
1516
1517Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Register Op0) {
1518 if ((!Subtarget->isLittleEndian())) {
1519 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1520 }
1521 return Register();
1522}
1523
1524Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Register Op0) {
1525 if ((!Subtarget->isLittleEndian())) {
1526 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1527 }
1528 return Register();
1529}
1530
1531Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Register Op0) {
1532 if ((!Subtarget->isLittleEndian())) {
1533 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1534 }
1535 return Register();
1536}
1537
1538Register fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, Register Op0) {
1539switch (RetVT.SimpleTy) {
1540 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0);
1541 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0);
1542 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0);
1543 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0);
1544 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0);
1545 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Op0);
1546 default: return Register();
1547}
1548}
1549
1550Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Register Op0) {
1551 if ((!Subtarget->isLittleEndian())) {
1552 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1553 }
1554 return Register();
1555}
1556
1557Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Register Op0) {
1558 if ((!Subtarget->isLittleEndian())) {
1559 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1560 }
1561 return Register();
1562}
1563
1564Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Register Op0) {
1565 if ((!Subtarget->isLittleEndian())) {
1566 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1567 }
1568 return Register();
1569}
1570
1571Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Register Op0) {
1572 if ((!Subtarget->isLittleEndian())) {
1573 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1574 }
1575 return Register();
1576}
1577
1578Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Register Op0) {
1579 if ((!Subtarget->isLittleEndian())) {
1580 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1581 }
1582 return Register();
1583}
1584
1585Register fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, Register Op0) {
1586switch (RetVT.SimpleTy) {
1587 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0);
1588 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0);
1589 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0);
1590 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0);
1591 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0);
1592 default: return Register();
1593}
1594}
1595
1596Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Register Op0) {
1597 if ((!Subtarget->isLittleEndian())) {
1598 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1599 }
1600 return Register();
1601}
1602
1603Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Register Op0) {
1604 if ((!Subtarget->isLittleEndian())) {
1605 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1606 }
1607 return Register();
1608}
1609
1610Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Register Op0) {
1611 if ((!Subtarget->isLittleEndian())) {
1612 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1613 }
1614 return Register();
1615}
1616
1617Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Register Op0) {
1618 if ((!Subtarget->isLittleEndian())) {
1619 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1620 }
1621 return Register();
1622}
1623
1624Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Register Op0) {
1625 if ((!Subtarget->isLittleEndian())) {
1626 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1627 }
1628 return Register();
1629}
1630
1631Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Register Op0) {
1632 if ((!Subtarget->isLittleEndian())) {
1633 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1634 }
1635 return Register();
1636}
1637
1638Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Register Op0) {
1639 if ((!Subtarget->isLittleEndian())) {
1640 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1641 }
1642 return Register();
1643}
1644
1645Register fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, Register Op0) {
1646switch (RetVT.SimpleTy) {
1647 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0);
1648 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0);
1649 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0);
1650 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0);
1651 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0);
1652 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Op0);
1653 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Op0);
1654 default: return Register();
1655}
1656}
1657
1658Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Register Op0) {
1659 if ((!Subtarget->isLittleEndian())) {
1660 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1661 }
1662 return Register();
1663}
1664
1665Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Register Op0) {
1666 if ((!Subtarget->isLittleEndian())) {
1667 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1668 }
1669 return Register();
1670}
1671
1672Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Register Op0) {
1673 if ((!Subtarget->isLittleEndian())) {
1674 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1675 }
1676 return Register();
1677}
1678
1679Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Register Op0) {
1680 if ((!Subtarget->isLittleEndian())) {
1681 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1682 }
1683 return Register();
1684}
1685
1686Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Register Op0) {
1687 if ((!Subtarget->isLittleEndian())) {
1688 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1689 }
1690 return Register();
1691}
1692
1693Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Register Op0) {
1694 if ((!Subtarget->isLittleEndian())) {
1695 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1696 }
1697 return Register();
1698}
1699
1700Register fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, Register Op0) {
1701switch (RetVT.SimpleTy) {
1702 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0);
1703 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0);
1704 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0);
1705 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0);
1706 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Op0);
1707 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0);
1708 default: return Register();
1709}
1710}
1711
1712Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Register Op0) {
1713 if ((!Subtarget->isLittleEndian())) {
1714 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1715 }
1716 return Register();
1717}
1718
1719Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Register Op0) {
1720 if ((!Subtarget->isLittleEndian())) {
1721 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1722 }
1723 return Register();
1724}
1725
1726Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Register Op0) {
1727 if ((!Subtarget->isLittleEndian())) {
1728 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1729 }
1730 return Register();
1731}
1732
1733Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Register Op0) {
1734 if ((!Subtarget->isLittleEndian())) {
1735 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1736 }
1737 return Register();
1738}
1739
1740Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Register Op0) {
1741 if ((!Subtarget->isLittleEndian())) {
1742 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1743 }
1744 return Register();
1745}
1746
1747Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Register Op0) {
1748 if ((!Subtarget->isLittleEndian())) {
1749 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1750 }
1751 return Register();
1752}
1753
1754Register fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, Register Op0) {
1755switch (RetVT.SimpleTy) {
1756 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0);
1757 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0);
1758 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0);
1759 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0);
1760 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Op0);
1761 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0);
1762 default: return Register();
1763}
1764}
1765
1766Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Register Op0) {
1767 if ((!Subtarget->isLittleEndian())) {
1768 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1769 }
1770 return Register();
1771}
1772
1773Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Register Op0) {
1774 if ((!Subtarget->isLittleEndian())) {
1775 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1776 }
1777 return Register();
1778}
1779
1780Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Register Op0) {
1781 if ((!Subtarget->isLittleEndian())) {
1782 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1783 }
1784 return Register();
1785}
1786
1787Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Register Op0) {
1788 if ((!Subtarget->isLittleEndian())) {
1789 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1790 }
1791 return Register();
1792}
1793
1794Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Register Op0) {
1795 if ((!Subtarget->isLittleEndian())) {
1796 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1797 }
1798 return Register();
1799}
1800
1801Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Register Op0) {
1802 if ((!Subtarget->isLittleEndian())) {
1803 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1804 }
1805 return Register();
1806}
1807
1808Register fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, Register Op0) {
1809switch (RetVT.SimpleTy) {
1810 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0);
1811 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0);
1812 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0);
1813 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0);
1814 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Op0);
1815 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0);
1816 default: return Register();
1817}
1818}
1819
1820Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Register Op0) {
1821 if ((!Subtarget->isLittleEndian())) {
1822 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1823 }
1824 return Register();
1825}
1826
1827Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Register Op0) {
1828 if ((!Subtarget->isLittleEndian())) {
1829 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1830 }
1831 return Register();
1832}
1833
1834Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Register Op0) {
1835 if ((!Subtarget->isLittleEndian())) {
1836 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1837 }
1838 return Register();
1839}
1840
1841Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Register Op0) {
1842 if ((!Subtarget->isLittleEndian())) {
1843 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1844 }
1845 return Register();
1846}
1847
1848Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Register Op0) {
1849 if ((!Subtarget->isLittleEndian())) {
1850 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1851 }
1852 return Register();
1853}
1854
1855Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Register Op0) {
1856 if ((!Subtarget->isLittleEndian())) {
1857 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1858 }
1859 return Register();
1860}
1861
1862Register fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, Register Op0) {
1863switch (RetVT.SimpleTy) {
1864 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0);
1865 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0);
1866 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0);
1867 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0);
1868 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0);
1869 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Op0);
1870 default: return Register();
1871}
1872}
1873
1874Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Register Op0) {
1875 if ((!Subtarget->isLittleEndian())) {
1876 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1877 }
1878 return Register();
1879}
1880
1881Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Register Op0) {
1882 if ((!Subtarget->isLittleEndian())) {
1883 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1884 }
1885 return Register();
1886}
1887
1888Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Register Op0) {
1889 if ((!Subtarget->isLittleEndian())) {
1890 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1891 }
1892 return Register();
1893}
1894
1895Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Register Op0) {
1896 if ((!Subtarget->isLittleEndian())) {
1897 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1898 }
1899 return Register();
1900}
1901
1902Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Register Op0) {
1903 if ((!Subtarget->isLittleEndian())) {
1904 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1905 }
1906 return Register();
1907}
1908
1909Register fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, Register Op0) {
1910switch (RetVT.SimpleTy) {
1911 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0);
1912 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0);
1913 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0);
1914 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0);
1915 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0);
1916 default: return Register();
1917}
1918}
1919
1920Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Register Op0) {
1921 if ((!Subtarget->isLittleEndian())) {
1922 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1923 }
1924 return Register();
1925}
1926
1927Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Register Op0) {
1928 if ((!Subtarget->isLittleEndian())) {
1929 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1930 }
1931 return Register();
1932}
1933
1934Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Register Op0) {
1935 if ((!Subtarget->isLittleEndian())) {
1936 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1937 }
1938 return Register();
1939}
1940
1941Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Register Op0) {
1942 if ((!Subtarget->isLittleEndian())) {
1943 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1944 }
1945 return Register();
1946}
1947
1948Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Register Op0) {
1949 if ((!Subtarget->isLittleEndian())) {
1950 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1951 }
1952 return Register();
1953}
1954
1955Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(Register Op0) {
1956 if ((!Subtarget->isLittleEndian())) {
1957 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1958 }
1959 return Register();
1960}
1961
1962Register fastEmit_ISD_BITCAST_MVT_v4bf16_r(MVT RetVT, Register Op0) {
1963switch (RetVT.SimpleTy) {
1964 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Op0);
1965 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Op0);
1966 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Op0);
1967 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Op0);
1968 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Op0);
1969 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(Op0);
1970 default: return Register();
1971}
1972}
1973
1974Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Register Op0) {
1975 if ((!Subtarget->isLittleEndian())) {
1976 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1977 }
1978 return Register();
1979}
1980
1981Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Register Op0) {
1982 if ((!Subtarget->isLittleEndian())) {
1983 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1984 }
1985 return Register();
1986}
1987
1988Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Register Op0) {
1989 if ((!Subtarget->isLittleEndian())) {
1990 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1991 }
1992 return Register();
1993}
1994
1995Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Register Op0) {
1996 if ((!Subtarget->isLittleEndian())) {
1997 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1998 }
1999 return Register();
2000}
2001
2002Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Register Op0) {
2003 if ((!Subtarget->isLittleEndian())) {
2004 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2005 }
2006 return Register();
2007}
2008
2009Register fastEmit_ISD_BITCAST_MVT_v8bf16_r(MVT RetVT, Register Op0) {
2010switch (RetVT.SimpleTy) {
2011 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Op0);
2012 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Op0);
2013 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Op0);
2014 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Op0);
2015 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Op0);
2016 default: return Register();
2017}
2018}
2019
2020Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Register Op0) {
2021 if ((!Subtarget->isLittleEndian())) {
2022 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2023 }
2024 return Register();
2025}
2026
2027Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Register Op0) {
2028 if ((!Subtarget->isLittleEndian())) {
2029 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
2030 }
2031 return Register();
2032}
2033
2034Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Register Op0) {
2035 if ((!Subtarget->isLittleEndian())) {
2036 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2037 }
2038 return Register();
2039}
2040
2041Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Register Op0) {
2042 if ((!Subtarget->isLittleEndian())) {
2043 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2044 }
2045 return Register();
2046}
2047
2048Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Register Op0) {
2049 if ((!Subtarget->isLittleEndian())) {
2050 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2051 }
2052 return Register();
2053}
2054
2055Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Register Op0) {
2056 if ((!Subtarget->isLittleEndian())) {
2057 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2058 }
2059 return Register();
2060}
2061
2062Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Register Op0) {
2063 if ((!Subtarget->isLittleEndian())) {
2064 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2065 }
2066 return Register();
2067}
2068
2069Register fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, Register Op0) {
2070switch (RetVT.SimpleTy) {
2071 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0);
2072 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0);
2073 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0);
2074 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0);
2075 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0);
2076 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Op0);
2077 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Op0);
2078 default: return Register();
2079}
2080}
2081
2082Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Register Op0) {
2083 if ((!Subtarget->isLittleEndian())) {
2084 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
2085 }
2086 return Register();
2087}
2088
2089Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Register Op0) {
2090 if ((!Subtarget->isLittleEndian())) {
2091 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2092 }
2093 return Register();
2094}
2095
2096Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Register Op0) {
2097 if ((!Subtarget->isLittleEndian())) {
2098 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2099 }
2100 return Register();
2101}
2102
2103Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Register Op0) {
2104 if ((!Subtarget->isLittleEndian())) {
2105 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2106 }
2107 return Register();
2108}
2109
2110Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Register Op0) {
2111 if ((!Subtarget->isLittleEndian())) {
2112 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2113 }
2114 return Register();
2115}
2116
2117Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Register Op0) {
2118 if ((!Subtarget->isLittleEndian())) {
2119 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2120 }
2121 return Register();
2122}
2123
2124Register fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, Register Op0) {
2125switch (RetVT.SimpleTy) {
2126 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0);
2127 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0);
2128 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0);
2129 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0);
2130 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Op0);
2131 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0);
2132 default: return Register();
2133}
2134}
2135
2136Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Register Op0) {
2137 if ((!Subtarget->isLittleEndian())) {
2138 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
2139 }
2140 return Register();
2141}
2142
2143Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Register Op0) {
2144 if ((!Subtarget->isLittleEndian())) {
2145 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2146 }
2147 return Register();
2148}
2149
2150Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Register Op0) {
2151 if ((!Subtarget->isLittleEndian())) {
2152 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2153 }
2154 return Register();
2155}
2156
2157Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Register Op0) {
2158 if ((!Subtarget->isLittleEndian())) {
2159 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2160 }
2161 return Register();
2162}
2163
2164Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(Register Op0) {
2165 if ((!Subtarget->isLittleEndian())) {
2166 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2167 }
2168 return Register();
2169}
2170
2171Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Register Op0) {
2172 if ((!Subtarget->isLittleEndian())) {
2173 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2174 }
2175 return Register();
2176}
2177
2178Register fastEmit_ISD_BITCAST_MVT_v1f64_r(MVT RetVT, Register Op0) {
2179switch (RetVT.SimpleTy) {
2180 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Op0);
2181 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Op0);
2182 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Op0);
2183 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Op0);
2184 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(Op0);
2185 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Op0);
2186 default: return Register();
2187}
2188}
2189
2190Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Register Op0) {
2191 if ((!Subtarget->isLittleEndian())) {
2192 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2193 }
2194 return Register();
2195}
2196
2197Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Register Op0) {
2198 if ((!Subtarget->isLittleEndian())) {
2199 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2200 }
2201 return Register();
2202}
2203
2204Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Register Op0) {
2205 if ((!Subtarget->isLittleEndian())) {
2206 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2207 }
2208 return Register();
2209}
2210
2211Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Register Op0) {
2212 if ((!Subtarget->isLittleEndian())) {
2213 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2214 }
2215 return Register();
2216}
2217
2218Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Register Op0) {
2219 if ((!Subtarget->isLittleEndian())) {
2220 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2221 }
2222 return Register();
2223}
2224
2225Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Register Op0) {
2226 if ((!Subtarget->isLittleEndian())) {
2227 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2228 }
2229 return Register();
2230}
2231
2232Register fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, Register Op0) {
2233switch (RetVT.SimpleTy) {
2234 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0);
2235 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0);
2236 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0);
2237 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0);
2238 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Op0);
2239 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0);
2240 default: return Register();
2241}
2242}
2243
2244Register fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, Register Op0) {
2245 switch (VT.SimpleTy) {
2246 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
2247 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0);
2248 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0);
2249 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0);
2250 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0);
2251 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0);
2252 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0);
2253 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0);
2254 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0);
2255 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0);
2256 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0);
2257 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v4bf16_r(RetVT, Op0);
2258 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v8bf16_r(RetVT, Op0);
2259 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0);
2260 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0);
2261 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v1f64_r(RetVT, Op0);
2262 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0);
2263 default: return Register();
2264 }
2265}
2266
2267// FastEmit functions for ISD::BITREVERSE.
2268
2269Register fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, Register Op0) {
2270 if (RetVT.SimpleTy != MVT::i32)
2271 return Register();
2272 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITWr, RC: &AArch64::GPR32RegClass, Op0);
2273}
2274
2275Register fastEmit_ISD_BITREVERSE_MVT_i64_r(MVT RetVT, Register Op0) {
2276 if (RetVT.SimpleTy != MVT::i64)
2277 return Register();
2278 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITXr, RC: &AArch64::GPR64RegClass, Op0);
2279}
2280
2281Register fastEmit_ISD_BITREVERSE_MVT_v8i8_r(MVT RetVT, Register Op0) {
2282 if (RetVT.SimpleTy != MVT::v8i8)
2283 return Register();
2284 if ((Subtarget->isNeonAvailable())) {
2285 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITv8i8, RC: &AArch64::FPR64RegClass, Op0);
2286 }
2287 return Register();
2288}
2289
2290Register fastEmit_ISD_BITREVERSE_MVT_v16i8_r(MVT RetVT, Register Op0) {
2291 if (RetVT.SimpleTy != MVT::v16i8)
2292 return Register();
2293 if ((Subtarget->isNeonAvailable())) {
2294 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITv16i8, RC: &AArch64::FPR128RegClass, Op0);
2295 }
2296 return Register();
2297}
2298
2299Register fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, Register Op0) {
2300 switch (VT.SimpleTy) {
2301 case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0);
2302 case MVT::i64: return fastEmit_ISD_BITREVERSE_MVT_i64_r(RetVT, Op0);
2303 case MVT::v8i8: return fastEmit_ISD_BITREVERSE_MVT_v8i8_r(RetVT, Op0);
2304 case MVT::v16i8: return fastEmit_ISD_BITREVERSE_MVT_v16i8_r(RetVT, Op0);
2305 default: return Register();
2306 }
2307}
2308
2309// FastEmit functions for ISD::BRIND.
2310
2311Register fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, Register Op0) {
2312 if (RetVT.SimpleTy != MVT::isVoid)
2313 return Register();
2314 return fastEmitInst_r(MachineInstOpcode: AArch64::BR, RC: &AArch64::GPR64RegClass, Op0);
2315}
2316
2317Register fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, Register Op0) {
2318 switch (VT.SimpleTy) {
2319 case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0);
2320 default: return Register();
2321 }
2322}
2323
2324// FastEmit functions for ISD::BSWAP.
2325
2326Register fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, Register Op0) {
2327 if (RetVT.SimpleTy != MVT::i32)
2328 return Register();
2329 return fastEmitInst_r(MachineInstOpcode: AArch64::REVWr, RC: &AArch64::GPR32RegClass, Op0);
2330}
2331
2332Register fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, Register Op0) {
2333 if (RetVT.SimpleTy != MVT::i64)
2334 return Register();
2335 return fastEmitInst_r(MachineInstOpcode: AArch64::REVXr, RC: &AArch64::GPR64RegClass, Op0);
2336}
2337
2338Register fastEmit_ISD_BSWAP_MVT_v4i16_r(MVT RetVT, Register Op0) {
2339 if (RetVT.SimpleTy != MVT::v4i16)
2340 return Register();
2341 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
2342}
2343
2344Register fastEmit_ISD_BSWAP_MVT_v8i16_r(MVT RetVT, Register Op0) {
2345 if (RetVT.SimpleTy != MVT::v8i16)
2346 return Register();
2347 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
2348}
2349
2350Register fastEmit_ISD_BSWAP_MVT_v2i32_r(MVT RetVT, Register Op0) {
2351 if (RetVT.SimpleTy != MVT::v2i32)
2352 return Register();
2353 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
2354}
2355
2356Register fastEmit_ISD_BSWAP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2357 if (RetVT.SimpleTy != MVT::v4i32)
2358 return Register();
2359 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
2360}
2361
2362Register fastEmit_ISD_BSWAP_MVT_v2i64_r(MVT RetVT, Register Op0) {
2363 if (RetVT.SimpleTy != MVT::v2i64)
2364 return Register();
2365 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2366}
2367
2368Register fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, Register Op0) {
2369 switch (VT.SimpleTy) {
2370 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
2371 case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0);
2372 case MVT::v4i16: return fastEmit_ISD_BSWAP_MVT_v4i16_r(RetVT, Op0);
2373 case MVT::v8i16: return fastEmit_ISD_BSWAP_MVT_v8i16_r(RetVT, Op0);
2374 case MVT::v2i32: return fastEmit_ISD_BSWAP_MVT_v2i32_r(RetVT, Op0);
2375 case MVT::v4i32: return fastEmit_ISD_BSWAP_MVT_v4i32_r(RetVT, Op0);
2376 case MVT::v2i64: return fastEmit_ISD_BSWAP_MVT_v2i64_r(RetVT, Op0);
2377 default: return Register();
2378 }
2379}
2380
2381// FastEmit functions for ISD::CTLS.
2382
2383Register fastEmit_ISD_CTLS_MVT_i32_r(MVT RetVT, Register Op0) {
2384 if (RetVT.SimpleTy != MVT::i32)
2385 return Register();
2386 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSWr, RC: &AArch64::GPR32RegClass, Op0);
2387}
2388
2389Register fastEmit_ISD_CTLS_MVT_i64_r(MVT RetVT, Register Op0) {
2390 if (RetVT.SimpleTy != MVT::i64)
2391 return Register();
2392 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSXr, RC: &AArch64::GPR64RegClass, Op0);
2393}
2394
2395Register fastEmit_ISD_CTLS_MVT_v8i8_r(MVT RetVT, Register Op0) {
2396 if (RetVT.SimpleTy != MVT::v8i8)
2397 return Register();
2398 if ((Subtarget->isNeonAvailable())) {
2399 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv8i8, RC: &AArch64::FPR64RegClass, Op0);
2400 }
2401 return Register();
2402}
2403
2404Register fastEmit_ISD_CTLS_MVT_v16i8_r(MVT RetVT, Register Op0) {
2405 if (RetVT.SimpleTy != MVT::v16i8)
2406 return Register();
2407 if ((Subtarget->isNeonAvailable())) {
2408 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv16i8, RC: &AArch64::FPR128RegClass, Op0);
2409 }
2410 return Register();
2411}
2412
2413Register fastEmit_ISD_CTLS_MVT_v4i16_r(MVT RetVT, Register Op0) {
2414 if (RetVT.SimpleTy != MVT::v4i16)
2415 return Register();
2416 if ((Subtarget->isNeonAvailable())) {
2417 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv4i16, RC: &AArch64::FPR64RegClass, Op0);
2418 }
2419 return Register();
2420}
2421
2422Register fastEmit_ISD_CTLS_MVT_v8i16_r(MVT RetVT, Register Op0) {
2423 if (RetVT.SimpleTy != MVT::v8i16)
2424 return Register();
2425 if ((Subtarget->isNeonAvailable())) {
2426 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv8i16, RC: &AArch64::FPR128RegClass, Op0);
2427 }
2428 return Register();
2429}
2430
2431Register fastEmit_ISD_CTLS_MVT_v2i32_r(MVT RetVT, Register Op0) {
2432 if (RetVT.SimpleTy != MVT::v2i32)
2433 return Register();
2434 if ((Subtarget->isNeonAvailable())) {
2435 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv2i32, RC: &AArch64::FPR64RegClass, Op0);
2436 }
2437 return Register();
2438}
2439
2440Register fastEmit_ISD_CTLS_MVT_v4i32_r(MVT RetVT, Register Op0) {
2441 if (RetVT.SimpleTy != MVT::v4i32)
2442 return Register();
2443 if ((Subtarget->isNeonAvailable())) {
2444 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv4i32, RC: &AArch64::FPR128RegClass, Op0);
2445 }
2446 return Register();
2447}
2448
2449Register fastEmit_ISD_CTLS_r(MVT VT, MVT RetVT, Register Op0) {
2450 switch (VT.SimpleTy) {
2451 case MVT::i32: return fastEmit_ISD_CTLS_MVT_i32_r(RetVT, Op0);
2452 case MVT::i64: return fastEmit_ISD_CTLS_MVT_i64_r(RetVT, Op0);
2453 case MVT::v8i8: return fastEmit_ISD_CTLS_MVT_v8i8_r(RetVT, Op0);
2454 case MVT::v16i8: return fastEmit_ISD_CTLS_MVT_v16i8_r(RetVT, Op0);
2455 case MVT::v4i16: return fastEmit_ISD_CTLS_MVT_v4i16_r(RetVT, Op0);
2456 case MVT::v8i16: return fastEmit_ISD_CTLS_MVT_v8i16_r(RetVT, Op0);
2457 case MVT::v2i32: return fastEmit_ISD_CTLS_MVT_v2i32_r(RetVT, Op0);
2458 case MVT::v4i32: return fastEmit_ISD_CTLS_MVT_v4i32_r(RetVT, Op0);
2459 default: return Register();
2460 }
2461}
2462
2463// FastEmit functions for ISD::CTLZ.
2464
2465Register fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, Register Op0) {
2466 if (RetVT.SimpleTy != MVT::i32)
2467 return Register();
2468 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZWr, RC: &AArch64::GPR32RegClass, Op0);
2469}
2470
2471Register fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, Register Op0) {
2472 if (RetVT.SimpleTy != MVT::i64)
2473 return Register();
2474 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZXr, RC: &AArch64::GPR64RegClass, Op0);
2475}
2476
2477Register fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, Register Op0) {
2478 if (RetVT.SimpleTy != MVT::v8i8)
2479 return Register();
2480 if ((Subtarget->isNeonAvailable())) {
2481 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv8i8, RC: &AArch64::FPR64RegClass, Op0);
2482 }
2483 return Register();
2484}
2485
2486Register fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, Register Op0) {
2487 if (RetVT.SimpleTy != MVT::v16i8)
2488 return Register();
2489 if ((Subtarget->isNeonAvailable())) {
2490 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv16i8, RC: &AArch64::FPR128RegClass, Op0);
2491 }
2492 return Register();
2493}
2494
2495Register fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, Register Op0) {
2496 if (RetVT.SimpleTy != MVT::v4i16)
2497 return Register();
2498 if ((Subtarget->isNeonAvailable())) {
2499 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv4i16, RC: &AArch64::FPR64RegClass, Op0);
2500 }
2501 return Register();
2502}
2503
2504Register fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, Register Op0) {
2505 if (RetVT.SimpleTy != MVT::v8i16)
2506 return Register();
2507 if ((Subtarget->isNeonAvailable())) {
2508 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv8i16, RC: &AArch64::FPR128RegClass, Op0);
2509 }
2510 return Register();
2511}
2512
2513Register fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, Register Op0) {
2514 if (RetVT.SimpleTy != MVT::v2i32)
2515 return Register();
2516 if ((Subtarget->isNeonAvailable())) {
2517 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv2i32, RC: &AArch64::FPR64RegClass, Op0);
2518 }
2519 return Register();
2520}
2521
2522Register fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, Register Op0) {
2523 if (RetVT.SimpleTy != MVT::v4i32)
2524 return Register();
2525 if ((Subtarget->isNeonAvailable())) {
2526 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv4i32, RC: &AArch64::FPR128RegClass, Op0);
2527 }
2528 return Register();
2529}
2530
2531Register fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, Register Op0) {
2532 switch (VT.SimpleTy) {
2533 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
2534 case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0);
2535 case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0);
2536 case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0);
2537 case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0);
2538 case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0);
2539 case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0);
2540 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
2541 default: return Register();
2542 }
2543}
2544
2545// FastEmit functions for ISD::CTPOP.
2546
2547Register fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, Register Op0) {
2548 if (RetVT.SimpleTy != MVT::i32)
2549 return Register();
2550 if ((Subtarget->hasCSSC())) {
2551 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTWr, RC: &AArch64::GPR32RegClass, Op0);
2552 }
2553 return Register();
2554}
2555
2556Register fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, Register Op0) {
2557 if (RetVT.SimpleTy != MVT::i64)
2558 return Register();
2559 if ((Subtarget->hasCSSC())) {
2560 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTXr, RC: &AArch64::GPR64RegClass, Op0);
2561 }
2562 return Register();
2563}
2564
2565Register fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, Register Op0) {
2566 if (RetVT.SimpleTy != MVT::v8i8)
2567 return Register();
2568 if ((Subtarget->isNeonAvailable())) {
2569 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTv8i8, RC: &AArch64::FPR64RegClass, Op0);
2570 }
2571 return Register();
2572}
2573
2574Register fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, Register Op0) {
2575 if (RetVT.SimpleTy != MVT::v16i8)
2576 return Register();
2577 if ((Subtarget->isNeonAvailable())) {
2578 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTv16i8, RC: &AArch64::FPR128RegClass, Op0);
2579 }
2580 return Register();
2581}
2582
2583Register fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, Register Op0) {
2584 switch (VT.SimpleTy) {
2585 case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0);
2586 case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0);
2587 case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0);
2588 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
2589 default: return Register();
2590 }
2591}
2592
2593// FastEmit functions for ISD::CTTZ.
2594
2595Register fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, Register Op0) {
2596 if (RetVT.SimpleTy != MVT::i32)
2597 return Register();
2598 if ((Subtarget->hasCSSC())) {
2599 return fastEmitInst_r(MachineInstOpcode: AArch64::CTZWr, RC: &AArch64::GPR32RegClass, Op0);
2600 }
2601 return Register();
2602}
2603
2604Register fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, Register Op0) {
2605 if (RetVT.SimpleTy != MVT::i64)
2606 return Register();
2607 if ((Subtarget->hasCSSC())) {
2608 return fastEmitInst_r(MachineInstOpcode: AArch64::CTZXr, RC: &AArch64::GPR64RegClass, Op0);
2609 }
2610 return Register();
2611}
2612
2613Register fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, Register Op0) {
2614 switch (VT.SimpleTy) {
2615 case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0);
2616 case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0);
2617 default: return Register();
2618 }
2619}
2620
2621// FastEmit functions for ISD::FABS.
2622
2623Register fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, Register Op0) {
2624 if (RetVT.SimpleTy != MVT::f16)
2625 return Register();
2626 if ((Subtarget->hasFullFP16())) {
2627 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSHr, RC: &AArch64::FPR16RegClass, Op0);
2628 }
2629 return Register();
2630}
2631
2632Register fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, Register Op0) {
2633 if (RetVT.SimpleTy != MVT::f32)
2634 return Register();
2635 if ((Subtarget->hasFPARMv8())) {
2636 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSSr, RC: &AArch64::FPR32RegClass, Op0);
2637 }
2638 return Register();
2639}
2640
2641Register fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, Register Op0) {
2642 if (RetVT.SimpleTy != MVT::f64)
2643 return Register();
2644 if ((Subtarget->hasFPARMv8())) {
2645 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSDr, RC: &AArch64::FPR64RegClass, Op0);
2646 }
2647 return Register();
2648}
2649
2650Register fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, Register Op0) {
2651 if (RetVT.SimpleTy != MVT::v4f16)
2652 return Register();
2653 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2654 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f16, RC: &AArch64::FPR64RegClass, Op0);
2655 }
2656 return Register();
2657}
2658
2659Register fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, Register Op0) {
2660 if (RetVT.SimpleTy != MVT::v8f16)
2661 return Register();
2662 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2663 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv8f16, RC: &AArch64::FPR128RegClass, Op0);
2664 }
2665 return Register();
2666}
2667
2668Register fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, Register Op0) {
2669 if (RetVT.SimpleTy != MVT::v2f32)
2670 return Register();
2671 if ((Subtarget->isNeonAvailable())) {
2672 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv2f32, RC: &AArch64::FPR64RegClass, Op0);
2673 }
2674 return Register();
2675}
2676
2677Register fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, Register Op0) {
2678 if (RetVT.SimpleTy != MVT::v4f32)
2679 return Register();
2680 if ((Subtarget->isNeonAvailable())) {
2681 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f32, RC: &AArch64::FPR128RegClass, Op0);
2682 }
2683 return Register();
2684}
2685
2686Register fastEmit_ISD_FABS_MVT_v2f64_r(MVT RetVT, Register Op0) {
2687 if (RetVT.SimpleTy != MVT::v2f64)
2688 return Register();
2689 if ((Subtarget->isNeonAvailable())) {
2690 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv2f64, RC: &AArch64::FPR128RegClass, Op0);
2691 }
2692 return Register();
2693}
2694
2695Register fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, Register Op0) {
2696 switch (VT.SimpleTy) {
2697 case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0);
2698 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
2699 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
2700 case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0);
2701 case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0);
2702 case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0);
2703 case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0);
2704 case MVT::v2f64: return fastEmit_ISD_FABS_MVT_v2f64_r(RetVT, Op0);
2705 default: return Register();
2706 }
2707}
2708
2709// FastEmit functions for ISD::FCEIL.
2710
2711Register fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
2712 if (RetVT.SimpleTy != MVT::f16)
2713 return Register();
2714 if ((Subtarget->hasFullFP16())) {
2715 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPHr, RC: &AArch64::FPR16RegClass, Op0);
2716 }
2717 return Register();
2718}
2719
2720Register fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
2721 if (RetVT.SimpleTy != MVT::f32)
2722 return Register();
2723 if ((Subtarget->hasFPARMv8())) {
2724 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPSr, RC: &AArch64::FPR32RegClass, Op0);
2725 }
2726 return Register();
2727}
2728
2729Register fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
2730 if (RetVT.SimpleTy != MVT::f64)
2731 return Register();
2732 if ((Subtarget->hasFPARMv8())) {
2733 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPDr, RC: &AArch64::FPR64RegClass, Op0);
2734 }
2735 return Register();
2736}
2737
2738Register fastEmit_ISD_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
2739 if (RetVT.SimpleTy != MVT::v4f16)
2740 return Register();
2741 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2742 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f16, RC: &AArch64::FPR64RegClass, Op0);
2743 }
2744 return Register();
2745}
2746
2747Register fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
2748 if (RetVT.SimpleTy != MVT::v8f16)
2749 return Register();
2750 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2751 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv8f16, RC: &AArch64::FPR128RegClass, Op0);
2752 }
2753 return Register();
2754}
2755
2756Register fastEmit_ISD_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
2757 if (RetVT.SimpleTy != MVT::v2f32)
2758 return Register();
2759 if ((Subtarget->isNeonAvailable())) {
2760 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f32, RC: &AArch64::FPR64RegClass, Op0);
2761 }
2762 return Register();
2763}
2764
2765Register fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
2766 if (RetVT.SimpleTy != MVT::v4f32)
2767 return Register();
2768 if ((Subtarget->isNeonAvailable())) {
2769 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f32, RC: &AArch64::FPR128RegClass, Op0);
2770 }
2771 return Register();
2772}
2773
2774Register fastEmit_ISD_FCEIL_MVT_v2f64_r(MVT RetVT, Register Op0) {
2775 if (RetVT.SimpleTy != MVT::v2f64)
2776 return Register();
2777 if ((Subtarget->isNeonAvailable())) {
2778 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f64, RC: &AArch64::FPR128RegClass, Op0);
2779 }
2780 return Register();
2781}
2782
2783Register fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
2784 switch (VT.SimpleTy) {
2785 case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0);
2786 case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0);
2787 case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0);
2788 case MVT::v4f16: return fastEmit_ISD_FCEIL_MVT_v4f16_r(RetVT, Op0);
2789 case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0);
2790 case MVT::v2f32: return fastEmit_ISD_FCEIL_MVT_v2f32_r(RetVT, Op0);
2791 case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0);
2792 case MVT::v2f64: return fastEmit_ISD_FCEIL_MVT_v2f64_r(RetVT, Op0);
2793 default: return Register();
2794 }
2795}
2796
2797// FastEmit functions for ISD::FFLOOR.
2798
2799Register fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
2800 if (RetVT.SimpleTy != MVT::f16)
2801 return Register();
2802 if ((Subtarget->hasFullFP16())) {
2803 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMHr, RC: &AArch64::FPR16RegClass, Op0);
2804 }
2805 return Register();
2806}
2807
2808Register fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
2809 if (RetVT.SimpleTy != MVT::f32)
2810 return Register();
2811 if ((Subtarget->hasFPARMv8())) {
2812 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMSr, RC: &AArch64::FPR32RegClass, Op0);
2813 }
2814 return Register();
2815}
2816
2817Register fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
2818 if (RetVT.SimpleTy != MVT::f64)
2819 return Register();
2820 if ((Subtarget->hasFPARMv8())) {
2821 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMDr, RC: &AArch64::FPR64RegClass, Op0);
2822 }
2823 return Register();
2824}
2825
2826Register fastEmit_ISD_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
2827 if (RetVT.SimpleTy != MVT::v4f16)
2828 return Register();
2829 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2830 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f16, RC: &AArch64::FPR64RegClass, Op0);
2831 }
2832 return Register();
2833}
2834
2835Register fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
2836 if (RetVT.SimpleTy != MVT::v8f16)
2837 return Register();
2838 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2839 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv8f16, RC: &AArch64::FPR128RegClass, Op0);
2840 }
2841 return Register();
2842}
2843
2844Register fastEmit_ISD_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
2845 if (RetVT.SimpleTy != MVT::v2f32)
2846 return Register();
2847 if ((Subtarget->isNeonAvailable())) {
2848 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f32, RC: &AArch64::FPR64RegClass, Op0);
2849 }
2850 return Register();
2851}
2852
2853Register fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
2854 if (RetVT.SimpleTy != MVT::v4f32)
2855 return Register();
2856 if ((Subtarget->isNeonAvailable())) {
2857 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f32, RC: &AArch64::FPR128RegClass, Op0);
2858 }
2859 return Register();
2860}
2861
2862Register fastEmit_ISD_FFLOOR_MVT_v2f64_r(MVT RetVT, Register Op0) {
2863 if (RetVT.SimpleTy != MVT::v2f64)
2864 return Register();
2865 if ((Subtarget->isNeonAvailable())) {
2866 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f64, RC: &AArch64::FPR128RegClass, Op0);
2867 }
2868 return Register();
2869}
2870
2871Register fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
2872 switch (VT.SimpleTy) {
2873 case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0);
2874 case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0);
2875 case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0);
2876 case MVT::v4f16: return fastEmit_ISD_FFLOOR_MVT_v4f16_r(RetVT, Op0);
2877 case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0);
2878 case MVT::v2f32: return fastEmit_ISD_FFLOOR_MVT_v2f32_r(RetVT, Op0);
2879 case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0);
2880 case MVT::v2f64: return fastEmit_ISD_FFLOOR_MVT_v2f64_r(RetVT, Op0);
2881 default: return Register();
2882 }
2883}
2884
2885// FastEmit functions for ISD::FNEARBYINT.
2886
2887Register fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
2888 if (RetVT.SimpleTy != MVT::f16)
2889 return Register();
2890 if ((Subtarget->hasFullFP16())) {
2891 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIHr, RC: &AArch64::FPR16RegClass, Op0);
2892 }
2893 return Register();
2894}
2895
2896Register fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
2897 if (RetVT.SimpleTy != MVT::f32)
2898 return Register();
2899 if ((Subtarget->hasFPARMv8())) {
2900 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTISr, RC: &AArch64::FPR32RegClass, Op0);
2901 }
2902 return Register();
2903}
2904
2905Register fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
2906 if (RetVT.SimpleTy != MVT::f64)
2907 return Register();
2908 if ((Subtarget->hasFPARMv8())) {
2909 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIDr, RC: &AArch64::FPR64RegClass, Op0);
2910 }
2911 return Register();
2912}
2913
2914Register fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2915 if (RetVT.SimpleTy != MVT::v4f16)
2916 return Register();
2917 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2918 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f16, RC: &AArch64::FPR64RegClass, Op0);
2919 }
2920 return Register();
2921}
2922
2923Register fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2924 if (RetVT.SimpleTy != MVT::v8f16)
2925 return Register();
2926 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2927 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv8f16, RC: &AArch64::FPR128RegClass, Op0);
2928 }
2929 return Register();
2930}
2931
2932Register fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2933 if (RetVT.SimpleTy != MVT::v2f32)
2934 return Register();
2935 if ((Subtarget->isNeonAvailable())) {
2936 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f32, RC: &AArch64::FPR64RegClass, Op0);
2937 }
2938 return Register();
2939}
2940
2941Register fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2942 if (RetVT.SimpleTy != MVT::v4f32)
2943 return Register();
2944 if ((Subtarget->isNeonAvailable())) {
2945 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f32, RC: &AArch64::FPR128RegClass, Op0);
2946 }
2947 return Register();
2948}
2949
2950Register fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
2951 if (RetVT.SimpleTy != MVT::v2f64)
2952 return Register();
2953 if ((Subtarget->isNeonAvailable())) {
2954 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f64, RC: &AArch64::FPR128RegClass, Op0);
2955 }
2956 return Register();
2957}
2958
2959Register fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
2960 switch (VT.SimpleTy) {
2961 case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0);
2962 case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0);
2963 case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0);
2964 case MVT::v4f16: return fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(RetVT, Op0);
2965 case MVT::v8f16: return fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(RetVT, Op0);
2966 case MVT::v2f32: return fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(RetVT, Op0);
2967 case MVT::v4f32: return fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(RetVT, Op0);
2968 case MVT::v2f64: return fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(RetVT, Op0);
2969 default: return Register();
2970 }
2971}
2972
2973// FastEmit functions for ISD::FNEG.
2974
2975Register fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, Register Op0) {
2976 if (RetVT.SimpleTy != MVT::f16)
2977 return Register();
2978 if ((Subtarget->hasFullFP16())) {
2979 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGHr, RC: &AArch64::FPR16RegClass, Op0);
2980 }
2981 return Register();
2982}
2983
2984Register fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, Register Op0) {
2985 if (RetVT.SimpleTy != MVT::f32)
2986 return Register();
2987 if ((Subtarget->hasFPARMv8())) {
2988 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGSr, RC: &AArch64::FPR32RegClass, Op0);
2989 }
2990 return Register();
2991}
2992
2993Register fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, Register Op0) {
2994 if (RetVT.SimpleTy != MVT::f64)
2995 return Register();
2996 if ((Subtarget->hasFPARMv8())) {
2997 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGDr, RC: &AArch64::FPR64RegClass, Op0);
2998 }
2999 return Register();
3000}
3001
3002Register fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, Register Op0) {
3003 if (RetVT.SimpleTy != MVT::v4f16)
3004 return Register();
3005 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3006 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f16, RC: &AArch64::FPR64RegClass, Op0);
3007 }
3008 return Register();
3009}
3010
3011Register fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, Register Op0) {
3012 if (RetVT.SimpleTy != MVT::v8f16)
3013 return Register();
3014 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3015 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv8f16, RC: &AArch64::FPR128RegClass, Op0);
3016 }
3017 return Register();
3018}
3019
3020Register fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, Register Op0) {
3021 if (RetVT.SimpleTy != MVT::v2f32)
3022 return Register();
3023 if ((Subtarget->isNeonAvailable())) {
3024 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv2f32, RC: &AArch64::FPR64RegClass, Op0);
3025 }
3026 return Register();
3027}
3028
3029Register fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, Register Op0) {
3030 if (RetVT.SimpleTy != MVT::v4f32)
3031 return Register();
3032 if ((Subtarget->isNeonAvailable())) {
3033 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f32, RC: &AArch64::FPR128RegClass, Op0);
3034 }
3035 return Register();
3036}
3037
3038Register fastEmit_ISD_FNEG_MVT_v2f64_r(MVT RetVT, Register Op0) {
3039 if (RetVT.SimpleTy != MVT::v2f64)
3040 return Register();
3041 if ((Subtarget->isNeonAvailable())) {
3042 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv2f64, RC: &AArch64::FPR128RegClass, Op0);
3043 }
3044 return Register();
3045}
3046
3047Register fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, Register Op0) {
3048 switch (VT.SimpleTy) {
3049 case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0);
3050 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
3051 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
3052 case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0);
3053 case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0);
3054 case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0);
3055 case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0);
3056 case MVT::v2f64: return fastEmit_ISD_FNEG_MVT_v2f64_r(RetVT, Op0);
3057 default: return Register();
3058 }
3059}
3060
3061// FastEmit functions for ISD::FP_EXTEND.
3062
3063Register fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Register Op0) {
3064 if ((Subtarget->hasFPARMv8())) {
3065 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSHr, RC: &AArch64::FPR32RegClass, Op0);
3066 }
3067 return Register();
3068}
3069
3070Register fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Register Op0) {
3071 if ((Subtarget->hasFPARMv8())) {
3072 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDHr, RC: &AArch64::FPR64RegClass, Op0);
3073 }
3074 return Register();
3075}
3076
3077Register fastEmit_ISD_FP_EXTEND_MVT_f16_r(MVT RetVT, Register Op0) {
3078switch (RetVT.SimpleTy) {
3079 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Op0);
3080 case MVT::f64: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Op0);
3081 default: return Register();
3082}
3083}
3084
3085Register fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
3086 if (RetVT.SimpleTy != MVT::f64)
3087 return Register();
3088 if ((Subtarget->hasFPARMv8())) {
3089 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDSr, RC: &AArch64::FPR64RegClass, Op0);
3090 }
3091 return Register();
3092}
3093
3094Register fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3095 if (RetVT.SimpleTy != MVT::v4f32)
3096 return Register();
3097 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv4i16, RC: &AArch64::FPR128RegClass, Op0);
3098}
3099
3100Register fastEmit_ISD_FP_EXTEND_MVT_v4bf16_r(MVT RetVT, Register Op0) {
3101 if (RetVT.SimpleTy != MVT::v4f32)
3102 return Register();
3103 if ((Subtarget->isNeonAvailable())) {
3104 return fastEmitInst_r(MachineInstOpcode: AArch64::SHLLv4i16, RC: &AArch64::FPR128RegClass, Op0);
3105 }
3106 return Register();
3107}
3108
3109Register fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3110 if (RetVT.SimpleTy != MVT::v2f64)
3111 return Register();
3112 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv2i32, RC: &AArch64::FPR128RegClass, Op0);
3113}
3114
3115Register fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3116 switch (VT.SimpleTy) {
3117 case MVT::f16: return fastEmit_ISD_FP_EXTEND_MVT_f16_r(RetVT, Op0);
3118 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
3119 case MVT::v4f16: return fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
3120 case MVT::v4bf16: return fastEmit_ISD_FP_EXTEND_MVT_v4bf16_r(RetVT, Op0);
3121 case MVT::v2f32: return fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(RetVT, Op0);
3122 default: return Register();
3123 }
3124}
3125
3126// FastEmit functions for ISD::FP_ROUND.
3127
3128Register fastEmit_ISD_FP_ROUND_MVT_f32_MVT_bf16_r(Register Op0) {
3129 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
3130 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVT, RC: &AArch64::FPR16RegClass, Op0);
3131 }
3132 return Register();
3133}
3134
3135Register fastEmit_ISD_FP_ROUND_MVT_f32_MVT_f16_r(Register Op0) {
3136 if ((Subtarget->hasFPARMv8())) {
3137 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHSr, RC: &AArch64::FPR16RegClass, Op0);
3138 }
3139 return Register();
3140}
3141
3142Register fastEmit_ISD_FP_ROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3143switch (RetVT.SimpleTy) {
3144 case MVT::bf16: return fastEmit_ISD_FP_ROUND_MVT_f32_MVT_bf16_r(Op0);
3145 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f32_MVT_f16_r(Op0);
3146 default: return Register();
3147}
3148}
3149
3150Register fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Register Op0) {
3151 if ((Subtarget->hasFPARMv8())) {
3152 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHDr, RC: &AArch64::FPR16RegClass, Op0);
3153 }
3154 return Register();
3155}
3156
3157Register fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Register Op0) {
3158 if ((Subtarget->hasFPARMv8())) {
3159 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSDr, RC: &AArch64::FPR32RegClass, Op0);
3160 }
3161 return Register();
3162}
3163
3164Register fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3165switch (RetVT.SimpleTy) {
3166 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Op0);
3167 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Op0);
3168 default: return Register();
3169}
3170}
3171
3172Register fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Register Op0) {
3173 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
3174}
3175
3176Register fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Register Op0) {
3177 if ((Subtarget->hasBF16()) && (Subtarget->isNeonAvailable())) {
3178 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVTN, RC: &AArch64::FPR64RegClass, Op0);
3179 }
3180 return Register();
3181}
3182
3183Register fastEmit_ISD_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3184switch (RetVT.SimpleTy) {
3185 case MVT::v4f16: return fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Op0);
3186 case MVT::v4bf16: return fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Op0);
3187 default: return Register();
3188}
3189}
3190
3191Register fastEmit_ISD_FP_ROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
3192 if (RetVT.SimpleTy != MVT::v2f32)
3193 return Register();
3194 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
3195}
3196
3197Register fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
3198 switch (VT.SimpleTy) {
3199 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f32_r(RetVT, Op0);
3200 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
3201 case MVT::v4f32: return fastEmit_ISD_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
3202 case MVT::v2f64: return fastEmit_ISD_FP_ROUND_MVT_v2f64_r(RetVT, Op0);
3203 default: return Register();
3204 }
3205}
3206
3207// FastEmit functions for ISD::FP_TO_SINT.
3208
3209Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
3210 if ((Subtarget->hasFPRCVT())) {
3211 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSHr, RC: &AArch64::FPR32RegClass, Op0);
3212 }
3213 if ((Subtarget->hasFullFP16())) {
3214 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
3215 }
3216 return Register();
3217}
3218
3219Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
3220 if ((Subtarget->hasFPRCVT())) {
3221 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDHr, RC: &AArch64::FPR64RegClass, Op0);
3222 }
3223 if ((Subtarget->hasFullFP16())) {
3224 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
3225 }
3226 return Register();
3227}
3228
3229Register fastEmit_ISD_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
3230switch (RetVT.SimpleTy) {
3231 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
3232 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
3233 default: return Register();
3234}
3235}
3236
3237Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
3238 if ((Subtarget->hasFPARMv8())) {
3239 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
3240 }
3241 return Register();
3242}
3243
3244Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
3245 if ((Subtarget->hasFPRCVT())) {
3246 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDSr, RC: &AArch64::FPR64RegClass, Op0);
3247 }
3248 if ((Subtarget->hasFPARMv8())) {
3249 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
3250 }
3251 return Register();
3252}
3253
3254Register fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
3255switch (RetVT.SimpleTy) {
3256 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
3257 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
3258 default: return Register();
3259}
3260}
3261
3262Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
3263 if ((Subtarget->hasFPRCVT())) {
3264 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSDr, RC: &AArch64::FPR32RegClass, Op0);
3265 }
3266 if ((Subtarget->hasFPARMv8())) {
3267 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
3268 }
3269 return Register();
3270}
3271
3272Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
3273 if ((Subtarget->hasFPARMv8())) {
3274 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
3275 }
3276 return Register();
3277}
3278
3279Register fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
3280switch (RetVT.SimpleTy) {
3281 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
3282 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
3283 default: return Register();
3284}
3285}
3286
3287Register fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3288 if (RetVT.SimpleTy != MVT::v4i16)
3289 return Register();
3290 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3291 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
3292 }
3293 return Register();
3294}
3295
3296Register fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3297 if (RetVT.SimpleTy != MVT::v8i16)
3298 return Register();
3299 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3300 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
3301 }
3302 return Register();
3303}
3304
3305Register fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3306 if (RetVT.SimpleTy != MVT::v2i32)
3307 return Register();
3308 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3309 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
3310 }
3311 return Register();
3312}
3313
3314Register fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3315 if (RetVT.SimpleTy != MVT::v4i32)
3316 return Register();
3317 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3318 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
3319 }
3320 return Register();
3321}
3322
3323Register fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3324 if (RetVT.SimpleTy != MVT::v2i64)
3325 return Register();
3326 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3327 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
3328 }
3329 return Register();
3330}
3331
3332Register fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
3333 switch (VT.SimpleTy) {
3334 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
3335 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
3336 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
3337 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
3338 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
3339 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
3340 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
3341 case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0);
3342 default: return Register();
3343 }
3344}
3345
3346// FastEmit functions for ISD::FP_TO_SINT_SAT.
3347
3348Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_MVT_i32_r(Register Op0) {
3349 if ((Subtarget->hasFPRCVT())) {
3350 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSHr, RC: &AArch64::FPR32RegClass, Op0);
3351 }
3352 if ((Subtarget->hasFullFP16())) {
3353 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
3354 }
3355 return Register();
3356}
3357
3358Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_MVT_i64_r(Register Op0) {
3359 if ((Subtarget->hasFPRCVT())) {
3360 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDHr, RC: &AArch64::FPR64RegClass, Op0);
3361 }
3362 if ((Subtarget->hasFullFP16())) {
3363 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
3364 }
3365 return Register();
3366}
3367
3368Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_r(MVT RetVT, Register Op0) {
3369switch (RetVT.SimpleTy) {
3370 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_MVT_i32_r(Op0);
3371 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_MVT_i64_r(Op0);
3372 default: return Register();
3373}
3374}
3375
3376Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_MVT_i32_r(Register Op0) {
3377 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
3378}
3379
3380Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_MVT_i64_r(Register Op0) {
3381 if ((Subtarget->hasFPRCVT())) {
3382 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDSr, RC: &AArch64::FPR64RegClass, Op0);
3383 }
3384 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
3385}
3386
3387Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_r(MVT RetVT, Register Op0) {
3388switch (RetVT.SimpleTy) {
3389 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_MVT_i32_r(Op0);
3390 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_MVT_i64_r(Op0);
3391 default: return Register();
3392}
3393}
3394
3395Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_MVT_i32_r(Register Op0) {
3396 if ((Subtarget->hasFPRCVT())) {
3397 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSDr, RC: &AArch64::FPR32RegClass, Op0);
3398 }
3399 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
3400}
3401
3402Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_MVT_i64_r(Register Op0) {
3403 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
3404}
3405
3406Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_r(MVT RetVT, Register Op0) {
3407switch (RetVT.SimpleTy) {
3408 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_MVT_i32_r(Op0);
3409 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_MVT_i64_r(Op0);
3410 default: return Register();
3411}
3412}
3413
3414Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3415 if (RetVT.SimpleTy != MVT::v4i16)
3416 return Register();
3417 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3418 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
3419 }
3420 return Register();
3421}
3422
3423Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3424 if (RetVT.SimpleTy != MVT::v8i16)
3425 return Register();
3426 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3427 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
3428 }
3429 return Register();
3430}
3431
3432Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3433 if (RetVT.SimpleTy != MVT::v2i32)
3434 return Register();
3435 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3436 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
3437 }
3438 return Register();
3439}
3440
3441Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3442 if (RetVT.SimpleTy != MVT::v4i32)
3443 return Register();
3444 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3445 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
3446 }
3447 return Register();
3448}
3449
3450Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3451 if (RetVT.SimpleTy != MVT::v2i64)
3452 return Register();
3453 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3454 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
3455 }
3456 return Register();
3457}
3458
3459Register fastEmit_ISD_FP_TO_SINT_SAT_r(MVT VT, MVT RetVT, Register Op0) {
3460 switch (VT.SimpleTy) {
3461 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_r(RetVT, Op0);
3462 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_r(RetVT, Op0);
3463 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_r(RetVT, Op0);
3464 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_v4f16_r(RetVT, Op0);
3465 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_v8f16_r(RetVT, Op0);
3466 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_v2f32_r(RetVT, Op0);
3467 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_v4f32_r(RetVT, Op0);
3468 case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_v2f64_r(RetVT, Op0);
3469 default: return Register();
3470 }
3471}
3472
3473// FastEmit functions for ISD::FP_TO_UINT.
3474
3475Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
3476 if ((Subtarget->hasFPRCVT())) {
3477 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSHr, RC: &AArch64::FPR32RegClass, Op0);
3478 }
3479 if ((Subtarget->hasFullFP16())) {
3480 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
3481 }
3482 return Register();
3483}
3484
3485Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
3486 if ((Subtarget->hasFPRCVT())) {
3487 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDHr, RC: &AArch64::FPR64RegClass, Op0);
3488 }
3489 if ((Subtarget->hasFullFP16())) {
3490 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
3491 }
3492 return Register();
3493}
3494
3495Register fastEmit_ISD_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
3496switch (RetVT.SimpleTy) {
3497 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
3498 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
3499 default: return Register();
3500}
3501}
3502
3503Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
3504 if ((Subtarget->hasFPARMv8())) {
3505 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
3506 }
3507 return Register();
3508}
3509
3510Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
3511 if ((Subtarget->hasFPRCVT())) {
3512 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDSr, RC: &AArch64::FPR64RegClass, Op0);
3513 }
3514 if ((Subtarget->hasFPARMv8())) {
3515 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
3516 }
3517 return Register();
3518}
3519
3520Register fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
3521switch (RetVT.SimpleTy) {
3522 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
3523 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
3524 default: return Register();
3525}
3526}
3527
3528Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
3529 if ((Subtarget->hasFPRCVT())) {
3530 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSDr, RC: &AArch64::FPR32RegClass, Op0);
3531 }
3532 if ((Subtarget->hasFPARMv8())) {
3533 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
3534 }
3535 return Register();
3536}
3537
3538Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
3539 if ((Subtarget->hasFPARMv8())) {
3540 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
3541 }
3542 return Register();
3543}
3544
3545Register fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
3546switch (RetVT.SimpleTy) {
3547 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
3548 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
3549 default: return Register();
3550}
3551}
3552
3553Register fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3554 if (RetVT.SimpleTy != MVT::v4i16)
3555 return Register();
3556 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3557 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
3558 }
3559 return Register();
3560}
3561
3562Register fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3563 if (RetVT.SimpleTy != MVT::v8i16)
3564 return Register();
3565 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3566 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
3567 }
3568 return Register();
3569}
3570
3571Register fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3572 if (RetVT.SimpleTy != MVT::v2i32)
3573 return Register();
3574 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3575 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
3576 }
3577 return Register();
3578}
3579
3580Register fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3581 if (RetVT.SimpleTy != MVT::v4i32)
3582 return Register();
3583 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3584 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
3585 }
3586 return Register();
3587}
3588
3589Register fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3590 if (RetVT.SimpleTy != MVT::v2i64)
3591 return Register();
3592 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3593 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
3594 }
3595 return Register();
3596}
3597
3598Register fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
3599 switch (VT.SimpleTy) {
3600 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
3601 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
3602 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
3603 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
3604 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
3605 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
3606 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
3607 case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0);
3608 default: return Register();
3609 }
3610}
3611
3612// FastEmit functions for ISD::FP_TO_UINT_SAT.
3613
3614Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_MVT_i32_r(Register Op0) {
3615 if ((Subtarget->hasFPRCVT())) {
3616 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSHr, RC: &AArch64::FPR32RegClass, Op0);
3617 }
3618 if ((Subtarget->hasFullFP16())) {
3619 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
3620 }
3621 return Register();
3622}
3623
3624Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_MVT_i64_r(Register Op0) {
3625 if ((Subtarget->hasFPRCVT())) {
3626 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDHr, RC: &AArch64::FPR64RegClass, Op0);
3627 }
3628 if ((Subtarget->hasFullFP16())) {
3629 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
3630 }
3631 return Register();
3632}
3633
3634Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_r(MVT RetVT, Register Op0) {
3635switch (RetVT.SimpleTy) {
3636 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_MVT_i32_r(Op0);
3637 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_MVT_i64_r(Op0);
3638 default: return Register();
3639}
3640}
3641
3642Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_MVT_i32_r(Register Op0) {
3643 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
3644}
3645
3646Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_MVT_i64_r(Register Op0) {
3647 if ((Subtarget->hasFPRCVT())) {
3648 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDSr, RC: &AArch64::FPR64RegClass, Op0);
3649 }
3650 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
3651}
3652
3653Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_r(MVT RetVT, Register Op0) {
3654switch (RetVT.SimpleTy) {
3655 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_MVT_i32_r(Op0);
3656 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_MVT_i64_r(Op0);
3657 default: return Register();
3658}
3659}
3660
3661Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_MVT_i32_r(Register Op0) {
3662 if ((Subtarget->hasFPRCVT())) {
3663 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSDr, RC: &AArch64::FPR32RegClass, Op0);
3664 }
3665 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
3666}
3667
3668Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_MVT_i64_r(Register Op0) {
3669 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
3670}
3671
3672Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_r(MVT RetVT, Register Op0) {
3673switch (RetVT.SimpleTy) {
3674 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_MVT_i32_r(Op0);
3675 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_MVT_i64_r(Op0);
3676 default: return Register();
3677}
3678}
3679
3680Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3681 if (RetVT.SimpleTy != MVT::v4i16)
3682 return Register();
3683 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3684 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
3685 }
3686 return Register();
3687}
3688
3689Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3690 if (RetVT.SimpleTy != MVT::v8i16)
3691 return Register();
3692 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3693 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
3694 }
3695 return Register();
3696}
3697
3698Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3699 if (RetVT.SimpleTy != MVT::v2i32)
3700 return Register();
3701 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3702 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
3703 }
3704 return Register();
3705}
3706
3707Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3708 if (RetVT.SimpleTy != MVT::v4i32)
3709 return Register();
3710 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3711 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
3712 }
3713 return Register();
3714}
3715
3716Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3717 if (RetVT.SimpleTy != MVT::v2i64)
3718 return Register();
3719 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3720 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
3721 }
3722 return Register();
3723}
3724
3725Register fastEmit_ISD_FP_TO_UINT_SAT_r(MVT VT, MVT RetVT, Register Op0) {
3726 switch (VT.SimpleTy) {
3727 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_r(RetVT, Op0);
3728 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_r(RetVT, Op0);
3729 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_r(RetVT, Op0);
3730 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_v4f16_r(RetVT, Op0);
3731 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_v8f16_r(RetVT, Op0);
3732 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_v2f32_r(RetVT, Op0);
3733 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_v4f32_r(RetVT, Op0);
3734 case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_v2f64_r(RetVT, Op0);
3735 default: return Register();
3736 }
3737}
3738
3739// FastEmit functions for ISD::FRINT.
3740
3741Register fastEmit_ISD_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
3742 if (RetVT.SimpleTy != MVT::f16)
3743 return Register();
3744 if ((Subtarget->hasFullFP16())) {
3745 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXHr, RC: &AArch64::FPR16RegClass, Op0);
3746 }
3747 return Register();
3748}
3749
3750Register fastEmit_ISD_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
3751 if (RetVT.SimpleTy != MVT::f32)
3752 return Register();
3753 if ((Subtarget->hasFPARMv8())) {
3754 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXSr, RC: &AArch64::FPR32RegClass, Op0);
3755 }
3756 return Register();
3757}
3758
3759Register fastEmit_ISD_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
3760 if (RetVT.SimpleTy != MVT::f64)
3761 return Register();
3762 if ((Subtarget->hasFPARMv8())) {
3763 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXDr, RC: &AArch64::FPR64RegClass, Op0);
3764 }
3765 return Register();
3766}
3767
3768Register fastEmit_ISD_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3769 if (RetVT.SimpleTy != MVT::v4f16)
3770 return Register();
3771 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3772 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f16, RC: &AArch64::FPR64RegClass, Op0);
3773 }
3774 return Register();
3775}
3776
3777Register fastEmit_ISD_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3778 if (RetVT.SimpleTy != MVT::v8f16)
3779 return Register();
3780 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3781 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv8f16, RC: &AArch64::FPR128RegClass, Op0);
3782 }
3783 return Register();
3784}
3785
3786Register fastEmit_ISD_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3787 if (RetVT.SimpleTy != MVT::v2f32)
3788 return Register();
3789 if ((Subtarget->isNeonAvailable())) {
3790 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f32, RC: &AArch64::FPR64RegClass, Op0);
3791 }
3792 return Register();
3793}
3794
3795Register fastEmit_ISD_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3796 if (RetVT.SimpleTy != MVT::v4f32)
3797 return Register();
3798 if ((Subtarget->isNeonAvailable())) {
3799 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f32, RC: &AArch64::FPR128RegClass, Op0);
3800 }
3801 return Register();
3802}
3803
3804Register fastEmit_ISD_FRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3805 if (RetVT.SimpleTy != MVT::v2f64)
3806 return Register();
3807 if ((Subtarget->isNeonAvailable())) {
3808 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f64, RC: &AArch64::FPR128RegClass, Op0);
3809 }
3810 return Register();
3811}
3812
3813Register fastEmit_ISD_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
3814 switch (VT.SimpleTy) {
3815 case MVT::f16: return fastEmit_ISD_FRINT_MVT_f16_r(RetVT, Op0);
3816 case MVT::f32: return fastEmit_ISD_FRINT_MVT_f32_r(RetVT, Op0);
3817 case MVT::f64: return fastEmit_ISD_FRINT_MVT_f64_r(RetVT, Op0);
3818 case MVT::v4f16: return fastEmit_ISD_FRINT_MVT_v4f16_r(RetVT, Op0);
3819 case MVT::v8f16: return fastEmit_ISD_FRINT_MVT_v8f16_r(RetVT, Op0);
3820 case MVT::v2f32: return fastEmit_ISD_FRINT_MVT_v2f32_r(RetVT, Op0);
3821 case MVT::v4f32: return fastEmit_ISD_FRINT_MVT_v4f32_r(RetVT, Op0);
3822 case MVT::v2f64: return fastEmit_ISD_FRINT_MVT_v2f64_r(RetVT, Op0);
3823 default: return Register();
3824 }
3825}
3826
3827// FastEmit functions for ISD::FROUND.
3828
3829Register fastEmit_ISD_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
3830 if (RetVT.SimpleTy != MVT::f16)
3831 return Register();
3832 if ((Subtarget->hasFullFP16())) {
3833 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAHr, RC: &AArch64::FPR16RegClass, Op0);
3834 }
3835 return Register();
3836}
3837
3838Register fastEmit_ISD_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3839 if (RetVT.SimpleTy != MVT::f32)
3840 return Register();
3841 if ((Subtarget->hasFPARMv8())) {
3842 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTASr, RC: &AArch64::FPR32RegClass, Op0);
3843 }
3844 return Register();
3845}
3846
3847Register fastEmit_ISD_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3848 if (RetVT.SimpleTy != MVT::f64)
3849 return Register();
3850 if ((Subtarget->hasFPARMv8())) {
3851 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTADr, RC: &AArch64::FPR64RegClass, Op0);
3852 }
3853 return Register();
3854}
3855
3856Register fastEmit_ISD_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3857 if (RetVT.SimpleTy != MVT::v4f16)
3858 return Register();
3859 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3860 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f16, RC: &AArch64::FPR64RegClass, Op0);
3861 }
3862 return Register();
3863}
3864
3865Register fastEmit_ISD_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
3866 if (RetVT.SimpleTy != MVT::v8f16)
3867 return Register();
3868 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3869 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv8f16, RC: &AArch64::FPR128RegClass, Op0);
3870 }
3871 return Register();
3872}
3873
3874Register fastEmit_ISD_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3875 if (RetVT.SimpleTy != MVT::v2f32)
3876 return Register();
3877 if ((Subtarget->isNeonAvailable())) {
3878 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f32, RC: &AArch64::FPR64RegClass, Op0);
3879 }
3880 return Register();
3881}
3882
3883Register fastEmit_ISD_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3884 if (RetVT.SimpleTy != MVT::v4f32)
3885 return Register();
3886 if ((Subtarget->isNeonAvailable())) {
3887 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f32, RC: &AArch64::FPR128RegClass, Op0);
3888 }
3889 return Register();
3890}
3891
3892Register fastEmit_ISD_FROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
3893 if (RetVT.SimpleTy != MVT::v2f64)
3894 return Register();
3895 if ((Subtarget->isNeonAvailable())) {
3896 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f64, RC: &AArch64::FPR128RegClass, Op0);
3897 }
3898 return Register();
3899}
3900
3901Register fastEmit_ISD_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
3902 switch (VT.SimpleTy) {
3903 case MVT::f16: return fastEmit_ISD_FROUND_MVT_f16_r(RetVT, Op0);
3904 case MVT::f32: return fastEmit_ISD_FROUND_MVT_f32_r(RetVT, Op0);
3905 case MVT::f64: return fastEmit_ISD_FROUND_MVT_f64_r(RetVT, Op0);
3906 case MVT::v4f16: return fastEmit_ISD_FROUND_MVT_v4f16_r(RetVT, Op0);
3907 case MVT::v8f16: return fastEmit_ISD_FROUND_MVT_v8f16_r(RetVT, Op0);
3908 case MVT::v2f32: return fastEmit_ISD_FROUND_MVT_v2f32_r(RetVT, Op0);
3909 case MVT::v4f32: return fastEmit_ISD_FROUND_MVT_v4f32_r(RetVT, Op0);
3910 case MVT::v2f64: return fastEmit_ISD_FROUND_MVT_v2f64_r(RetVT, Op0);
3911 default: return Register();
3912 }
3913}
3914
3915// FastEmit functions for ISD::FROUNDEVEN.
3916
3917Register fastEmit_ISD_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
3918 if (RetVT.SimpleTy != MVT::f16)
3919 return Register();
3920 if ((Subtarget->hasFullFP16())) {
3921 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNHr, RC: &AArch64::FPR16RegClass, Op0);
3922 }
3923 return Register();
3924}
3925
3926Register fastEmit_ISD_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
3927 if (RetVT.SimpleTy != MVT::f32)
3928 return Register();
3929 if ((Subtarget->hasFPARMv8())) {
3930 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNSr, RC: &AArch64::FPR32RegClass, Op0);
3931 }
3932 return Register();
3933}
3934
3935Register fastEmit_ISD_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
3936 if (RetVT.SimpleTy != MVT::f64)
3937 return Register();
3938 if ((Subtarget->hasFPARMv8())) {
3939 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNDr, RC: &AArch64::FPR64RegClass, Op0);
3940 }
3941 return Register();
3942}
3943
3944Register fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
3945 if (RetVT.SimpleTy != MVT::v4f16)
3946 return Register();
3947 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3948 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f16, RC: &AArch64::FPR64RegClass, Op0);
3949 }
3950 return Register();
3951}
3952
3953Register fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
3954 if (RetVT.SimpleTy != MVT::v8f16)
3955 return Register();
3956 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3957 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv8f16, RC: &AArch64::FPR128RegClass, Op0);
3958 }
3959 return Register();
3960}
3961
3962Register fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
3963 if (RetVT.SimpleTy != MVT::v2f32)
3964 return Register();
3965 if ((Subtarget->isNeonAvailable())) {
3966 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f32, RC: &AArch64::FPR64RegClass, Op0);
3967 }
3968 return Register();
3969}
3970
3971Register fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
3972 if (RetVT.SimpleTy != MVT::v4f32)
3973 return Register();
3974 if ((Subtarget->isNeonAvailable())) {
3975 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f32, RC: &AArch64::FPR128RegClass, Op0);
3976 }
3977 return Register();
3978}
3979
3980Register fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, Register Op0) {
3981 if (RetVT.SimpleTy != MVT::v2f64)
3982 return Register();
3983 if ((Subtarget->isNeonAvailable())) {
3984 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f64, RC: &AArch64::FPR128RegClass, Op0);
3985 }
3986 return Register();
3987}
3988
3989Register fastEmit_ISD_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
3990 switch (VT.SimpleTy) {
3991 case MVT::f16: return fastEmit_ISD_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
3992 case MVT::f32: return fastEmit_ISD_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
3993 case MVT::f64: return fastEmit_ISD_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
3994 case MVT::v4f16: return fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
3995 case MVT::v8f16: return fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
3996 case MVT::v2f32: return fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
3997 case MVT::v4f32: return fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
3998 case MVT::v2f64: return fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0);
3999 default: return Register();
4000 }
4001}
4002
4003// FastEmit functions for ISD::FSQRT.
4004
4005Register fastEmit_ISD_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
4006 if (RetVT.SimpleTy != MVT::f16)
4007 return Register();
4008 if ((Subtarget->hasFullFP16())) {
4009 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTHr, RC: &AArch64::FPR16RegClass, Op0);
4010 }
4011 return Register();
4012}
4013
4014Register fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
4015 if (RetVT.SimpleTy != MVT::f32)
4016 return Register();
4017 if ((Subtarget->hasFPARMv8())) {
4018 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTSr, RC: &AArch64::FPR32RegClass, Op0);
4019 }
4020 return Register();
4021}
4022
4023Register fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
4024 if (RetVT.SimpleTy != MVT::f64)
4025 return Register();
4026 if ((Subtarget->hasFPARMv8())) {
4027 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTDr, RC: &AArch64::FPR64RegClass, Op0);
4028 }
4029 return Register();
4030}
4031
4032Register fastEmit_ISD_FSQRT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4033 if (RetVT.SimpleTy != MVT::v4f16)
4034 return Register();
4035 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4036 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f16, RC: &AArch64::FPR64RegClass, Op0);
4037 }
4038 return Register();
4039}
4040
4041Register fastEmit_ISD_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4042 if (RetVT.SimpleTy != MVT::v8f16)
4043 return Register();
4044 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4045 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv8f16, RC: &AArch64::FPR128RegClass, Op0);
4046 }
4047 return Register();
4048}
4049
4050Register fastEmit_ISD_FSQRT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4051 if (RetVT.SimpleTy != MVT::v2f32)
4052 return Register();
4053 if ((Subtarget->isNeonAvailable())) {
4054 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f32, RC: &AArch64::FPR64RegClass, Op0);
4055 }
4056 return Register();
4057}
4058
4059Register fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4060 if (RetVT.SimpleTy != MVT::v4f32)
4061 return Register();
4062 if ((Subtarget->isNeonAvailable())) {
4063 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f32, RC: &AArch64::FPR128RegClass, Op0);
4064 }
4065 return Register();
4066}
4067
4068Register fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4069 if (RetVT.SimpleTy != MVT::v2f64)
4070 return Register();
4071 if ((Subtarget->isNeonAvailable())) {
4072 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f64, RC: &AArch64::FPR128RegClass, Op0);
4073 }
4074 return Register();
4075}
4076
4077Register fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
4078 switch (VT.SimpleTy) {
4079 case MVT::f16: return fastEmit_ISD_FSQRT_MVT_f16_r(RetVT, Op0);
4080 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
4081 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
4082 case MVT::v4f16: return fastEmit_ISD_FSQRT_MVT_v4f16_r(RetVT, Op0);
4083 case MVT::v8f16: return fastEmit_ISD_FSQRT_MVT_v8f16_r(RetVT, Op0);
4084 case MVT::v2f32: return fastEmit_ISD_FSQRT_MVT_v2f32_r(RetVT, Op0);
4085 case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0);
4086 case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0);
4087 default: return Register();
4088 }
4089}
4090
4091// FastEmit functions for ISD::FTRUNC.
4092
4093Register fastEmit_ISD_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
4094 if (RetVT.SimpleTy != MVT::f16)
4095 return Register();
4096 if ((Subtarget->hasFullFP16())) {
4097 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZHr, RC: &AArch64::FPR16RegClass, Op0);
4098 }
4099 return Register();
4100}
4101
4102Register fastEmit_ISD_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
4103 if (RetVT.SimpleTy != MVT::f32)
4104 return Register();
4105 if ((Subtarget->hasFPARMv8())) {
4106 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZSr, RC: &AArch64::FPR32RegClass, Op0);
4107 }
4108 return Register();
4109}
4110
4111Register fastEmit_ISD_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
4112 if (RetVT.SimpleTy != MVT::f64)
4113 return Register();
4114 if ((Subtarget->hasFPARMv8())) {
4115 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZDr, RC: &AArch64::FPR64RegClass, Op0);
4116 }
4117 return Register();
4118}
4119
4120Register fastEmit_ISD_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
4121 if (RetVT.SimpleTy != MVT::v4f16)
4122 return Register();
4123 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4124 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f16, RC: &AArch64::FPR64RegClass, Op0);
4125 }
4126 return Register();
4127}
4128
4129Register fastEmit_ISD_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
4130 if (RetVT.SimpleTy != MVT::v8f16)
4131 return Register();
4132 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4133 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv8f16, RC: &AArch64::FPR128RegClass, Op0);
4134 }
4135 return Register();
4136}
4137
4138Register fastEmit_ISD_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
4139 if (RetVT.SimpleTy != MVT::v2f32)
4140 return Register();
4141 if ((Subtarget->isNeonAvailable())) {
4142 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f32, RC: &AArch64::FPR64RegClass, Op0);
4143 }
4144 return Register();
4145}
4146
4147Register fastEmit_ISD_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
4148 if (RetVT.SimpleTy != MVT::v4f32)
4149 return Register();
4150 if ((Subtarget->isNeonAvailable())) {
4151 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f32, RC: &AArch64::FPR128RegClass, Op0);
4152 }
4153 return Register();
4154}
4155
4156Register fastEmit_ISD_FTRUNC_MVT_v2f64_r(MVT RetVT, Register Op0) {
4157 if (RetVT.SimpleTy != MVT::v2f64)
4158 return Register();
4159 if ((Subtarget->isNeonAvailable())) {
4160 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f64, RC: &AArch64::FPR128RegClass, Op0);
4161 }
4162 return Register();
4163}
4164
4165Register fastEmit_ISD_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
4166 switch (VT.SimpleTy) {
4167 case MVT::f16: return fastEmit_ISD_FTRUNC_MVT_f16_r(RetVT, Op0);
4168 case MVT::f32: return fastEmit_ISD_FTRUNC_MVT_f32_r(RetVT, Op0);
4169 case MVT::f64: return fastEmit_ISD_FTRUNC_MVT_f64_r(RetVT, Op0);
4170 case MVT::v4f16: return fastEmit_ISD_FTRUNC_MVT_v4f16_r(RetVT, Op0);
4171 case MVT::v8f16: return fastEmit_ISD_FTRUNC_MVT_v8f16_r(RetVT, Op0);
4172 case MVT::v2f32: return fastEmit_ISD_FTRUNC_MVT_v2f32_r(RetVT, Op0);
4173 case MVT::v4f32: return fastEmit_ISD_FTRUNC_MVT_v4f32_r(RetVT, Op0);
4174 case MVT::v2f64: return fastEmit_ISD_FTRUNC_MVT_v2f64_r(RetVT, Op0);
4175 default: return Register();
4176 }
4177}
4178
4179// FastEmit functions for ISD::LLROUND.
4180
4181Register fastEmit_ISD_LLROUND_MVT_f16_r(MVT RetVT, Register Op0) {
4182 if (RetVT.SimpleTy != MVT::i64)
4183 return Register();
4184 if ((Subtarget->hasFPRCVT())) {
4185 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDHr, RC: &AArch64::FPR64RegClass, Op0);
4186 }
4187 if ((Subtarget->hasFullFP16())) {
4188 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
4189 }
4190 return Register();
4191}
4192
4193Register fastEmit_ISD_LLROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4194 if (RetVT.SimpleTy != MVT::i64)
4195 return Register();
4196 if ((Subtarget->hasFPRCVT())) {
4197 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDSr, RC: &AArch64::FPR64RegClass, Op0);
4198 }
4199 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
4200}
4201
4202Register fastEmit_ISD_LLROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4203 if (RetVT.SimpleTy != MVT::i64)
4204 return Register();
4205 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
4206}
4207
4208Register fastEmit_ISD_LLROUND_r(MVT VT, MVT RetVT, Register Op0) {
4209 switch (VT.SimpleTy) {
4210 case MVT::f16: return fastEmit_ISD_LLROUND_MVT_f16_r(RetVT, Op0);
4211 case MVT::f32: return fastEmit_ISD_LLROUND_MVT_f32_r(RetVT, Op0);
4212 case MVT::f64: return fastEmit_ISD_LLROUND_MVT_f64_r(RetVT, Op0);
4213 default: return Register();
4214 }
4215}
4216
4217// FastEmit functions for ISD::LROUND.
4218
4219Register fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(Register Op0) {
4220 if ((Subtarget->hasFPRCVT())) {
4221 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASSHr, RC: &AArch64::FPR32RegClass, Op0);
4222 }
4223 if ((Subtarget->hasFullFP16())) {
4224 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWHr, RC: &AArch64::GPR32RegClass, Op0);
4225 }
4226 return Register();
4227}
4228
4229Register fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(Register Op0) {
4230 if ((Subtarget->hasFPRCVT())) {
4231 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDHr, RC: &AArch64::FPR64RegClass, Op0);
4232 }
4233 if ((Subtarget->hasFullFP16())) {
4234 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
4235 }
4236 return Register();
4237}
4238
4239Register fastEmit_ISD_LROUND_MVT_f16_r(MVT RetVT, Register Op0) {
4240switch (RetVT.SimpleTy) {
4241 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(Op0);
4242 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(Op0);
4243 default: return Register();
4244}
4245}
4246
4247Register fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(Register Op0) {
4248 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWSr, RC: &AArch64::GPR32RegClass, Op0);
4249}
4250
4251Register fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(Register Op0) {
4252 if ((Subtarget->hasFPRCVT())) {
4253 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDSr, RC: &AArch64::FPR64RegClass, Op0);
4254 }
4255 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
4256}
4257
4258Register fastEmit_ISD_LROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4259switch (RetVT.SimpleTy) {
4260 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(Op0);
4261 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(Op0);
4262 default: return Register();
4263}
4264}
4265
4266Register fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(Register Op0) {
4267 if ((Subtarget->hasFPRCVT())) {
4268 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASSDr, RC: &AArch64::FPR32RegClass, Op0);
4269 }
4270 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWDr, RC: &AArch64::GPR32RegClass, Op0);
4271}
4272
4273Register fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(Register Op0) {
4274 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
4275}
4276
4277Register fastEmit_ISD_LROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4278switch (RetVT.SimpleTy) {
4279 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(Op0);
4280 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(Op0);
4281 default: return Register();
4282}
4283}
4284
4285Register fastEmit_ISD_LROUND_r(MVT VT, MVT RetVT, Register Op0) {
4286 switch (VT.SimpleTy) {
4287 case MVT::f16: return fastEmit_ISD_LROUND_MVT_f16_r(RetVT, Op0);
4288 case MVT::f32: return fastEmit_ISD_LROUND_MVT_f32_r(RetVT, Op0);
4289 case MVT::f64: return fastEmit_ISD_LROUND_MVT_f64_r(RetVT, Op0);
4290 default: return Register();
4291 }
4292}
4293
4294// FastEmit functions for ISD::SINT_TO_FP.
4295
4296Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
4297 if ((Subtarget->hasFPRCVT())) {
4298 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
4299 }
4300 if ((Subtarget->hasFullFP16())) {
4301 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
4302 }
4303 return Register();
4304}
4305
4306Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
4307 if ((Subtarget->hasFPARMv8())) {
4308 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
4309 }
4310 return Register();
4311}
4312
4313Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
4314 if ((Subtarget->hasFPRCVT())) {
4315 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
4316 }
4317 if ((Subtarget->hasFPARMv8())) {
4318 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
4319 }
4320 return Register();
4321}
4322
4323Register fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
4324switch (RetVT.SimpleTy) {
4325 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
4326 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
4327 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
4328 default: return Register();
4329}
4330}
4331
4332Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
4333 if ((Subtarget->hasFPRCVT())) {
4334 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
4335 }
4336 if ((Subtarget->hasFullFP16())) {
4337 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
4338 }
4339 return Register();
4340}
4341
4342Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
4343 if ((Subtarget->hasFPRCVT())) {
4344 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
4345 }
4346 if ((Subtarget->hasFPARMv8())) {
4347 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
4348 }
4349 return Register();
4350}
4351
4352Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
4353 if ((Subtarget->hasFPARMv8())) {
4354 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
4355 }
4356 return Register();
4357}
4358
4359Register fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
4360switch (RetVT.SimpleTy) {
4361 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
4362 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
4363 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
4364 default: return Register();
4365}
4366}
4367
4368Register fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
4369 if (RetVT.SimpleTy != MVT::v4f16)
4370 return Register();
4371 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4372 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
4373 }
4374 return Register();
4375}
4376
4377Register fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
4378 if (RetVT.SimpleTy != MVT::v8f16)
4379 return Register();
4380 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4381 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
4382 }
4383 return Register();
4384}
4385
4386Register fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
4387 if (RetVT.SimpleTy != MVT::v2f32)
4388 return Register();
4389 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4390 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
4391 }
4392 return Register();
4393}
4394
4395Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
4396 if (RetVT.SimpleTy != MVT::v4f32)
4397 return Register();
4398 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4399 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
4400 }
4401 return Register();
4402}
4403
4404Register fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
4405 if (RetVT.SimpleTy != MVT::v2f64)
4406 return Register();
4407 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4408 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
4409 }
4410 return Register();
4411}
4412
4413Register fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
4414 switch (VT.SimpleTy) {
4415 case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
4416 case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
4417 case MVT::v4i16: return fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
4418 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
4419 case MVT::v2i32: return fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
4420 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
4421 case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
4422 default: return Register();
4423 }
4424}
4425
4426// FastEmit functions for ISD::SPLAT_VECTOR.
4427
4428Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(Register Op0) {
4429 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4430 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_B, RC: &AArch64::ZPRRegClass, Op0);
4431 }
4432 return Register();
4433}
4434
4435Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(Register Op0) {
4436 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4437 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_H, RC: &AArch64::ZPRRegClass, Op0);
4438 }
4439 return Register();
4440}
4441
4442Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(Register Op0) {
4443 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4444 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_S, RC: &AArch64::ZPRRegClass, Op0);
4445 }
4446 return Register();
4447}
4448
4449Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(MVT RetVT, Register Op0) {
4450switch (RetVT.SimpleTy) {
4451 case MVT::nxv16i8: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(Op0);
4452 case MVT::nxv8i16: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(Op0);
4453 case MVT::nxv4i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(Op0);
4454 default: return Register();
4455}
4456}
4457
4458Register fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(MVT RetVT, Register Op0) {
4459 if (RetVT.SimpleTy != MVT::nxv2i64)
4460 return Register();
4461 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4462 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_D, RC: &AArch64::ZPRRegClass, Op0);
4463 }
4464 return Register();
4465}
4466
4467Register fastEmit_ISD_SPLAT_VECTOR_r(MVT VT, MVT RetVT, Register Op0) {
4468 switch (VT.SimpleTy) {
4469 case MVT::i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(RetVT, Op0);
4470 case MVT::i64: return fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(RetVT, Op0);
4471 default: return Register();
4472 }
4473}
4474
4475// FastEmit functions for ISD::STRICT_FCEIL.
4476
4477Register fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
4478 if (RetVT.SimpleTy != MVT::f16)
4479 return Register();
4480 if ((Subtarget->hasFullFP16())) {
4481 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPHr, RC: &AArch64::FPR16RegClass, Op0);
4482 }
4483 return Register();
4484}
4485
4486Register fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
4487 if (RetVT.SimpleTy != MVT::f32)
4488 return Register();
4489 if ((Subtarget->hasFPARMv8())) {
4490 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPSr, RC: &AArch64::FPR32RegClass, Op0);
4491 }
4492 return Register();
4493}
4494
4495Register fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
4496 if (RetVT.SimpleTy != MVT::f64)
4497 return Register();
4498 if ((Subtarget->hasFPARMv8())) {
4499 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPDr, RC: &AArch64::FPR64RegClass, Op0);
4500 }
4501 return Register();
4502}
4503
4504Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
4505 if (RetVT.SimpleTy != MVT::v4f16)
4506 return Register();
4507 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4508 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f16, RC: &AArch64::FPR64RegClass, Op0);
4509 }
4510 return Register();
4511}
4512
4513Register fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
4514 if (RetVT.SimpleTy != MVT::v8f16)
4515 return Register();
4516 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4517 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv8f16, RC: &AArch64::FPR128RegClass, Op0);
4518 }
4519 return Register();
4520}
4521
4522Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
4523 if (RetVT.SimpleTy != MVT::v2f32)
4524 return Register();
4525 if ((Subtarget->isNeonAvailable())) {
4526 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f32, RC: &AArch64::FPR64RegClass, Op0);
4527 }
4528 return Register();
4529}
4530
4531Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
4532 if (RetVT.SimpleTy != MVT::v4f32)
4533 return Register();
4534 if ((Subtarget->isNeonAvailable())) {
4535 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f32, RC: &AArch64::FPR128RegClass, Op0);
4536 }
4537 return Register();
4538}
4539
4540Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(MVT RetVT, Register Op0) {
4541 if (RetVT.SimpleTy != MVT::v2f64)
4542 return Register();
4543 if ((Subtarget->isNeonAvailable())) {
4544 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f64, RC: &AArch64::FPR128RegClass, Op0);
4545 }
4546 return Register();
4547}
4548
4549Register fastEmit_ISD_STRICT_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
4550 switch (VT.SimpleTy) {
4551 case MVT::f16: return fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(RetVT, Op0);
4552 case MVT::f32: return fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(RetVT, Op0);
4553 case MVT::f64: return fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(RetVT, Op0);
4554 case MVT::v4f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(RetVT, Op0);
4555 case MVT::v8f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(RetVT, Op0);
4556 case MVT::v2f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(RetVT, Op0);
4557 case MVT::v4f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(RetVT, Op0);
4558 case MVT::v2f64: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(RetVT, Op0);
4559 default: return Register();
4560 }
4561}
4562
4563// FastEmit functions for ISD::STRICT_FFLOOR.
4564
4565Register fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
4566 if (RetVT.SimpleTy != MVT::f16)
4567 return Register();
4568 if ((Subtarget->hasFullFP16())) {
4569 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMHr, RC: &AArch64::FPR16RegClass, Op0);
4570 }
4571 return Register();
4572}
4573
4574Register fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
4575 if (RetVT.SimpleTy != MVT::f32)
4576 return Register();
4577 if ((Subtarget->hasFPARMv8())) {
4578 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMSr, RC: &AArch64::FPR32RegClass, Op0);
4579 }
4580 return Register();
4581}
4582
4583Register fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
4584 if (RetVT.SimpleTy != MVT::f64)
4585 return Register();
4586 if ((Subtarget->hasFPARMv8())) {
4587 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMDr, RC: &AArch64::FPR64RegClass, Op0);
4588 }
4589 return Register();
4590}
4591
4592Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
4593 if (RetVT.SimpleTy != MVT::v4f16)
4594 return Register();
4595 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4596 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f16, RC: &AArch64::FPR64RegClass, Op0);
4597 }
4598 return Register();
4599}
4600
4601Register fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
4602 if (RetVT.SimpleTy != MVT::v8f16)
4603 return Register();
4604 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4605 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv8f16, RC: &AArch64::FPR128RegClass, Op0);
4606 }
4607 return Register();
4608}
4609
4610Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
4611 if (RetVT.SimpleTy != MVT::v2f32)
4612 return Register();
4613 if ((Subtarget->isNeonAvailable())) {
4614 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f32, RC: &AArch64::FPR64RegClass, Op0);
4615 }
4616 return Register();
4617}
4618
4619Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
4620 if (RetVT.SimpleTy != MVT::v4f32)
4621 return Register();
4622 if ((Subtarget->isNeonAvailable())) {
4623 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f32, RC: &AArch64::FPR128RegClass, Op0);
4624 }
4625 return Register();
4626}
4627
4628Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(MVT RetVT, Register Op0) {
4629 if (RetVT.SimpleTy != MVT::v2f64)
4630 return Register();
4631 if ((Subtarget->isNeonAvailable())) {
4632 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f64, RC: &AArch64::FPR128RegClass, Op0);
4633 }
4634 return Register();
4635}
4636
4637Register fastEmit_ISD_STRICT_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
4638 switch (VT.SimpleTy) {
4639 case MVT::f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(RetVT, Op0);
4640 case MVT::f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(RetVT, Op0);
4641 case MVT::f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(RetVT, Op0);
4642 case MVT::v4f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(RetVT, Op0);
4643 case MVT::v8f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(RetVT, Op0);
4644 case MVT::v2f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(RetVT, Op0);
4645 case MVT::v4f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(RetVT, Op0);
4646 case MVT::v2f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(RetVT, Op0);
4647 default: return Register();
4648 }
4649}
4650
4651// FastEmit functions for ISD::STRICT_FNEARBYINT.
4652
4653Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
4654 if (RetVT.SimpleTy != MVT::f16)
4655 return Register();
4656 if ((Subtarget->hasFullFP16())) {
4657 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIHr, RC: &AArch64::FPR16RegClass, Op0);
4658 }
4659 return Register();
4660}
4661
4662Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
4663 if (RetVT.SimpleTy != MVT::f32)
4664 return Register();
4665 if ((Subtarget->hasFPARMv8())) {
4666 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTISr, RC: &AArch64::FPR32RegClass, Op0);
4667 }
4668 return Register();
4669}
4670
4671Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
4672 if (RetVT.SimpleTy != MVT::f64)
4673 return Register();
4674 if ((Subtarget->hasFPARMv8())) {
4675 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIDr, RC: &AArch64::FPR64RegClass, Op0);
4676 }
4677 return Register();
4678}
4679
4680Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4681 if (RetVT.SimpleTy != MVT::v4f16)
4682 return Register();
4683 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4684 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f16, RC: &AArch64::FPR64RegClass, Op0);
4685 }
4686 return Register();
4687}
4688
4689Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4690 if (RetVT.SimpleTy != MVT::v8f16)
4691 return Register();
4692 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4693 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv8f16, RC: &AArch64::FPR128RegClass, Op0);
4694 }
4695 return Register();
4696}
4697
4698Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4699 if (RetVT.SimpleTy != MVT::v2f32)
4700 return Register();
4701 if ((Subtarget->isNeonAvailable())) {
4702 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f32, RC: &AArch64::FPR64RegClass, Op0);
4703 }
4704 return Register();
4705}
4706
4707Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4708 if (RetVT.SimpleTy != MVT::v4f32)
4709 return Register();
4710 if ((Subtarget->isNeonAvailable())) {
4711 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f32, RC: &AArch64::FPR128RegClass, Op0);
4712 }
4713 return Register();
4714}
4715
4716Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4717 if (RetVT.SimpleTy != MVT::v2f64)
4718 return Register();
4719 if ((Subtarget->isNeonAvailable())) {
4720 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f64, RC: &AArch64::FPR128RegClass, Op0);
4721 }
4722 return Register();
4723}
4724
4725Register fastEmit_ISD_STRICT_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
4726 switch (VT.SimpleTy) {
4727 case MVT::f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(RetVT, Op0);
4728 case MVT::f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(RetVT, Op0);
4729 case MVT::f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(RetVT, Op0);
4730 case MVT::v4f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(RetVT, Op0);
4731 case MVT::v8f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(RetVT, Op0);
4732 case MVT::v2f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(RetVT, Op0);
4733 case MVT::v4f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(RetVT, Op0);
4734 case MVT::v2f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(RetVT, Op0);
4735 default: return Register();
4736 }
4737}
4738
4739// FastEmit functions for ISD::STRICT_FP_EXTEND.
4740
4741Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(Register Op0) {
4742 if ((Subtarget->hasFPARMv8())) {
4743 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSHr, RC: &AArch64::FPR32RegClass, Op0);
4744 }
4745 return Register();
4746}
4747
4748Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(Register Op0) {
4749 if ((Subtarget->hasFPARMv8())) {
4750 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDHr, RC: &AArch64::FPR64RegClass, Op0);
4751 }
4752 return Register();
4753}
4754
4755Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(MVT RetVT, Register Op0) {
4756switch (RetVT.SimpleTy) {
4757 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(Op0);
4758 case MVT::f64: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(Op0);
4759 default: return Register();
4760}
4761}
4762
4763Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
4764 if (RetVT.SimpleTy != MVT::f64)
4765 return Register();
4766 if ((Subtarget->hasFPARMv8())) {
4767 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDSr, RC: &AArch64::FPR64RegClass, Op0);
4768 }
4769 return Register();
4770}
4771
4772Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
4773 if (RetVT.SimpleTy != MVT::v4f32)
4774 return Register();
4775 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv4i16, RC: &AArch64::FPR128RegClass, Op0);
4776}
4777
4778Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4bf16_r(MVT RetVT, Register Op0) {
4779 if (RetVT.SimpleTy != MVT::v4f32)
4780 return Register();
4781 if ((Subtarget->isNeonAvailable())) {
4782 return fastEmitInst_r(MachineInstOpcode: AArch64::SHLLv4i16, RC: &AArch64::FPR128RegClass, Op0);
4783 }
4784 return Register();
4785}
4786
4787Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(MVT RetVT, Register Op0) {
4788 if (RetVT.SimpleTy != MVT::v2f64)
4789 return Register();
4790 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv2i32, RC: &AArch64::FPR128RegClass, Op0);
4791}
4792
4793Register fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
4794 switch (VT.SimpleTy) {
4795 case MVT::f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(RetVT, Op0);
4796 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0);
4797 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
4798 case MVT::v4bf16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4bf16_r(RetVT, Op0);
4799 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(RetVT, Op0);
4800 default: return Register();
4801 }
4802}
4803
4804// FastEmit functions for ISD::STRICT_FP_ROUND.
4805
4806Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_bf16_r(Register Op0) {
4807 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
4808 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVT, RC: &AArch64::FPR16RegClass, Op0);
4809 }
4810 return Register();
4811}
4812
4813Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_f16_r(Register Op0) {
4814 if ((Subtarget->hasFPARMv8())) {
4815 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHSr, RC: &AArch64::FPR16RegClass, Op0);
4816 }
4817 return Register();
4818}
4819
4820Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4821switch (RetVT.SimpleTy) {
4822 case MVT::bf16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_bf16_r(Op0);
4823 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_f16_r(Op0);
4824 default: return Register();
4825}
4826}
4827
4828Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(Register Op0) {
4829 if ((Subtarget->hasFPARMv8())) {
4830 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHDr, RC: &AArch64::FPR16RegClass, Op0);
4831 }
4832 return Register();
4833}
4834
4835Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(Register Op0) {
4836 if ((Subtarget->hasFPARMv8())) {
4837 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSDr, RC: &AArch64::FPR32RegClass, Op0);
4838 }
4839 return Register();
4840}
4841
4842Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4843switch (RetVT.SimpleTy) {
4844 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(Op0);
4845 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(Op0);
4846 default: return Register();
4847}
4848}
4849
4850Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Register Op0) {
4851 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
4852}
4853
4854Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Register Op0) {
4855 if ((Subtarget->hasBF16()) && (Subtarget->isNeonAvailable())) {
4856 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVTN, RC: &AArch64::FPR64RegClass, Op0);
4857 }
4858 return Register();
4859}
4860
4861Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
4862switch (RetVT.SimpleTy) {
4863 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Op0);
4864 case MVT::v4bf16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Op0);
4865 default: return Register();
4866}
4867}
4868
4869Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
4870 if (RetVT.SimpleTy != MVT::v2f32)
4871 return Register();
4872 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
4873}
4874
4875Register fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
4876 switch (VT.SimpleTy) {
4877 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(RetVT, Op0);
4878 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0);
4879 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
4880 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(RetVT, Op0);
4881 default: return Register();
4882 }
4883}
4884
4885// FastEmit functions for ISD::STRICT_FP_TO_SINT.
4886
4887Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
4888 if ((Subtarget->hasFPRCVT())) {
4889 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSHr, RC: &AArch64::FPR32RegClass, Op0);
4890 }
4891 if ((Subtarget->hasFullFP16())) {
4892 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
4893 }
4894 return Register();
4895}
4896
4897Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
4898 if ((Subtarget->hasFPRCVT())) {
4899 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDHr, RC: &AArch64::FPR64RegClass, Op0);
4900 }
4901 if ((Subtarget->hasFullFP16())) {
4902 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
4903 }
4904 return Register();
4905}
4906
4907Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
4908switch (RetVT.SimpleTy) {
4909 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
4910 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
4911 default: return Register();
4912}
4913}
4914
4915Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
4916 if ((Subtarget->hasFPARMv8())) {
4917 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
4918 }
4919 return Register();
4920}
4921
4922Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
4923 if ((Subtarget->hasFPRCVT())) {
4924 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDSr, RC: &AArch64::FPR64RegClass, Op0);
4925 }
4926 if ((Subtarget->hasFPARMv8())) {
4927 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
4928 }
4929 return Register();
4930}
4931
4932Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
4933switch (RetVT.SimpleTy) {
4934 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
4935 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
4936 default: return Register();
4937}
4938}
4939
4940Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
4941 if ((Subtarget->hasFPRCVT())) {
4942 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSDr, RC: &AArch64::FPR32RegClass, Op0);
4943 }
4944 if ((Subtarget->hasFPARMv8())) {
4945 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
4946 }
4947 return Register();
4948}
4949
4950Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
4951 if ((Subtarget->hasFPARMv8())) {
4952 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
4953 }
4954 return Register();
4955}
4956
4957Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
4958switch (RetVT.SimpleTy) {
4959 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
4960 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
4961 default: return Register();
4962}
4963}
4964
4965Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4966 if (RetVT.SimpleTy != MVT::v4i16)
4967 return Register();
4968 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4969 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
4970 }
4971 return Register();
4972}
4973
4974Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4975 if (RetVT.SimpleTy != MVT::v8i16)
4976 return Register();
4977 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4978 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
4979 }
4980 return Register();
4981}
4982
4983Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4984 if (RetVT.SimpleTy != MVT::v2i32)
4985 return Register();
4986 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4987 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
4988 }
4989 return Register();
4990}
4991
4992Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4993 if (RetVT.SimpleTy != MVT::v4i32)
4994 return Register();
4995 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4996 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
4997 }
4998 return Register();
4999}
5000
5001Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
5002 if (RetVT.SimpleTy != MVT::v2i64)
5003 return Register();
5004 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5005 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
5006 }
5007 return Register();
5008}
5009
5010Register fastEmit_ISD_STRICT_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
5011 switch (VT.SimpleTy) {
5012 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
5013 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
5014 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
5015 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
5016 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
5017 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
5018 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
5019 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0);
5020 default: return Register();
5021 }
5022}
5023
5024// FastEmit functions for ISD::STRICT_FP_TO_UINT.
5025
5026Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
5027 if ((Subtarget->hasFPRCVT())) {
5028 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSHr, RC: &AArch64::FPR32RegClass, Op0);
5029 }
5030 if ((Subtarget->hasFullFP16())) {
5031 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
5032 }
5033 return Register();
5034}
5035
5036Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
5037 if ((Subtarget->hasFPRCVT())) {
5038 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDHr, RC: &AArch64::FPR64RegClass, Op0);
5039 }
5040 if ((Subtarget->hasFullFP16())) {
5041 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
5042 }
5043 return Register();
5044}
5045
5046Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
5047switch (RetVT.SimpleTy) {
5048 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
5049 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
5050 default: return Register();
5051}
5052}
5053
5054Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
5055 if ((Subtarget->hasFPARMv8())) {
5056 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
5057 }
5058 return Register();
5059}
5060
5061Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
5062 if ((Subtarget->hasFPRCVT())) {
5063 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDSr, RC: &AArch64::FPR64RegClass, Op0);
5064 }
5065 if ((Subtarget->hasFPARMv8())) {
5066 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
5067 }
5068 return Register();
5069}
5070
5071Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
5072switch (RetVT.SimpleTy) {
5073 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
5074 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
5075 default: return Register();
5076}
5077}
5078
5079Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
5080 if ((Subtarget->hasFPRCVT())) {
5081 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSDr, RC: &AArch64::FPR32RegClass, Op0);
5082 }
5083 if ((Subtarget->hasFPARMv8())) {
5084 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
5085 }
5086 return Register();
5087}
5088
5089Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
5090 if ((Subtarget->hasFPARMv8())) {
5091 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
5092 }
5093 return Register();
5094}
5095
5096Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
5097switch (RetVT.SimpleTy) {
5098 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
5099 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
5100 default: return Register();
5101}
5102}
5103
5104Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
5105 if (RetVT.SimpleTy != MVT::v4i16)
5106 return Register();
5107 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5108 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
5109 }
5110 return Register();
5111}
5112
5113Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
5114 if (RetVT.SimpleTy != MVT::v8i16)
5115 return Register();
5116 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5117 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
5118 }
5119 return Register();
5120}
5121
5122Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
5123 if (RetVT.SimpleTy != MVT::v2i32)
5124 return Register();
5125 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5126 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
5127 }
5128 return Register();
5129}
5130
5131Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
5132 if (RetVT.SimpleTy != MVT::v4i32)
5133 return Register();
5134 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5135 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
5136 }
5137 return Register();
5138}
5139
5140Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
5141 if (RetVT.SimpleTy != MVT::v2i64)
5142 return Register();
5143 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5144 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
5145 }
5146 return Register();
5147}
5148
5149Register fastEmit_ISD_STRICT_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
5150 switch (VT.SimpleTy) {
5151 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
5152 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
5153 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
5154 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
5155 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
5156 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
5157 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
5158 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0);
5159 default: return Register();
5160 }
5161}
5162
5163// FastEmit functions for ISD::STRICT_FRINT.
5164
5165Register fastEmit_ISD_STRICT_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
5166 if (RetVT.SimpleTy != MVT::f16)
5167 return Register();
5168 if ((Subtarget->hasFullFP16())) {
5169 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXHr, RC: &AArch64::FPR16RegClass, Op0);
5170 }
5171 return Register();
5172}
5173
5174Register fastEmit_ISD_STRICT_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
5175 if (RetVT.SimpleTy != MVT::f32)
5176 return Register();
5177 if ((Subtarget->hasFPARMv8())) {
5178 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXSr, RC: &AArch64::FPR32RegClass, Op0);
5179 }
5180 return Register();
5181}
5182
5183Register fastEmit_ISD_STRICT_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
5184 if (RetVT.SimpleTy != MVT::f64)
5185 return Register();
5186 if ((Subtarget->hasFPARMv8())) {
5187 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXDr, RC: &AArch64::FPR64RegClass, Op0);
5188 }
5189 return Register();
5190}
5191
5192Register fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
5193 if (RetVT.SimpleTy != MVT::v4f16)
5194 return Register();
5195 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5196 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f16, RC: &AArch64::FPR64RegClass, Op0);
5197 }
5198 return Register();
5199}
5200
5201Register fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
5202 if (RetVT.SimpleTy != MVT::v8f16)
5203 return Register();
5204 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5205 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv8f16, RC: &AArch64::FPR128RegClass, Op0);
5206 }
5207 return Register();
5208}
5209
5210Register fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
5211 if (RetVT.SimpleTy != MVT::v2f32)
5212 return Register();
5213 if ((Subtarget->isNeonAvailable())) {
5214 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f32, RC: &AArch64::FPR64RegClass, Op0);
5215 }
5216 return Register();
5217}
5218
5219Register fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
5220 if (RetVT.SimpleTy != MVT::v4f32)
5221 return Register();
5222 if ((Subtarget->isNeonAvailable())) {
5223 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f32, RC: &AArch64::FPR128RegClass, Op0);
5224 }
5225 return Register();
5226}
5227
5228Register fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
5229 if (RetVT.SimpleTy != MVT::v2f64)
5230 return Register();
5231 if ((Subtarget->isNeonAvailable())) {
5232 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f64, RC: &AArch64::FPR128RegClass, Op0);
5233 }
5234 return Register();
5235}
5236
5237Register fastEmit_ISD_STRICT_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
5238 switch (VT.SimpleTy) {
5239 case MVT::f16: return fastEmit_ISD_STRICT_FRINT_MVT_f16_r(RetVT, Op0);
5240 case MVT::f32: return fastEmit_ISD_STRICT_FRINT_MVT_f32_r(RetVT, Op0);
5241 case MVT::f64: return fastEmit_ISD_STRICT_FRINT_MVT_f64_r(RetVT, Op0);
5242 case MVT::v4f16: return fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(RetVT, Op0);
5243 case MVT::v8f16: return fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(RetVT, Op0);
5244 case MVT::v2f32: return fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(RetVT, Op0);
5245 case MVT::v4f32: return fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(RetVT, Op0);
5246 case MVT::v2f64: return fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(RetVT, Op0);
5247 default: return Register();
5248 }
5249}
5250
5251// FastEmit functions for ISD::STRICT_FROUND.
5252
5253Register fastEmit_ISD_STRICT_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5254 if (RetVT.SimpleTy != MVT::f16)
5255 return Register();
5256 if ((Subtarget->hasFullFP16())) {
5257 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAHr, RC: &AArch64::FPR16RegClass, Op0);
5258 }
5259 return Register();
5260}
5261
5262Register fastEmit_ISD_STRICT_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5263 if (RetVT.SimpleTy != MVT::f32)
5264 return Register();
5265 if ((Subtarget->hasFPARMv8())) {
5266 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTASr, RC: &AArch64::FPR32RegClass, Op0);
5267 }
5268 return Register();
5269}
5270
5271Register fastEmit_ISD_STRICT_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5272 if (RetVT.SimpleTy != MVT::f64)
5273 return Register();
5274 if ((Subtarget->hasFPARMv8())) {
5275 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTADr, RC: &AArch64::FPR64RegClass, Op0);
5276 }
5277 return Register();
5278}
5279
5280Register fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
5281 if (RetVT.SimpleTy != MVT::v4f16)
5282 return Register();
5283 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5284 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f16, RC: &AArch64::FPR64RegClass, Op0);
5285 }
5286 return Register();
5287}
5288
5289Register fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
5290 if (RetVT.SimpleTy != MVT::v8f16)
5291 return Register();
5292 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5293 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv8f16, RC: &AArch64::FPR128RegClass, Op0);
5294 }
5295 return Register();
5296}
5297
5298Register fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
5299 if (RetVT.SimpleTy != MVT::v2f32)
5300 return Register();
5301 if ((Subtarget->isNeonAvailable())) {
5302 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f32, RC: &AArch64::FPR64RegClass, Op0);
5303 }
5304 return Register();
5305}
5306
5307Register fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
5308 if (RetVT.SimpleTy != MVT::v4f32)
5309 return Register();
5310 if ((Subtarget->isNeonAvailable())) {
5311 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f32, RC: &AArch64::FPR128RegClass, Op0);
5312 }
5313 return Register();
5314}
5315
5316Register fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
5317 if (RetVT.SimpleTy != MVT::v2f64)
5318 return Register();
5319 if ((Subtarget->isNeonAvailable())) {
5320 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f64, RC: &AArch64::FPR128RegClass, Op0);
5321 }
5322 return Register();
5323}
5324
5325Register fastEmit_ISD_STRICT_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
5326 switch (VT.SimpleTy) {
5327 case MVT::f16: return fastEmit_ISD_STRICT_FROUND_MVT_f16_r(RetVT, Op0);
5328 case MVT::f32: return fastEmit_ISD_STRICT_FROUND_MVT_f32_r(RetVT, Op0);
5329 case MVT::f64: return fastEmit_ISD_STRICT_FROUND_MVT_f64_r(RetVT, Op0);
5330 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(RetVT, Op0);
5331 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(RetVT, Op0);
5332 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(RetVT, Op0);
5333 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(RetVT, Op0);
5334 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(RetVT, Op0);
5335 default: return Register();
5336 }
5337}
5338
5339// FastEmit functions for ISD::STRICT_FROUNDEVEN.
5340
5341Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
5342 if (RetVT.SimpleTy != MVT::f16)
5343 return Register();
5344 if ((Subtarget->hasFullFP16())) {
5345 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNHr, RC: &AArch64::FPR16RegClass, Op0);
5346 }
5347 return Register();
5348}
5349
5350Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
5351 if (RetVT.SimpleTy != MVT::f32)
5352 return Register();
5353 if ((Subtarget->hasFPARMv8())) {
5354 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNSr, RC: &AArch64::FPR32RegClass, Op0);
5355 }
5356 return Register();
5357}
5358
5359Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
5360 if (RetVT.SimpleTy != MVT::f64)
5361 return Register();
5362 if ((Subtarget->hasFPARMv8())) {
5363 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNDr, RC: &AArch64::FPR64RegClass, Op0);
5364 }
5365 return Register();
5366}
5367
5368Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
5369 if (RetVT.SimpleTy != MVT::v4f16)
5370 return Register();
5371 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5372 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f16, RC: &AArch64::FPR64RegClass, Op0);
5373 }
5374 return Register();
5375}
5376
5377Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
5378 if (RetVT.SimpleTy != MVT::v8f16)
5379 return Register();
5380 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5381 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv8f16, RC: &AArch64::FPR128RegClass, Op0);
5382 }
5383 return Register();
5384}
5385
5386Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
5387 if (RetVT.SimpleTy != MVT::v2f32)
5388 return Register();
5389 if ((Subtarget->isNeonAvailable())) {
5390 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f32, RC: &AArch64::FPR64RegClass, Op0);
5391 }
5392 return Register();
5393}
5394
5395Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
5396 if (RetVT.SimpleTy != MVT::v4f32)
5397 return Register();
5398 if ((Subtarget->isNeonAvailable())) {
5399 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f32, RC: &AArch64::FPR128RegClass, Op0);
5400 }
5401 return Register();
5402}
5403
5404Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, Register Op0) {
5405 if (RetVT.SimpleTy != MVT::v2f64)
5406 return Register();
5407 if ((Subtarget->isNeonAvailable())) {
5408 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f64, RC: &AArch64::FPR128RegClass, Op0);
5409 }
5410 return Register();
5411}
5412
5413Register fastEmit_ISD_STRICT_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
5414 switch (VT.SimpleTy) {
5415 case MVT::f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
5416 case MVT::f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
5417 case MVT::f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
5418 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
5419 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
5420 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
5421 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
5422 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0);
5423 default: return Register();
5424 }
5425}
5426
5427// FastEmit functions for ISD::STRICT_FSQRT.
5428
5429Register fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
5430 if (RetVT.SimpleTy != MVT::f16)
5431 return Register();
5432 if ((Subtarget->hasFullFP16())) {
5433 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTHr, RC: &AArch64::FPR16RegClass, Op0);
5434 }
5435 return Register();
5436}
5437
5438Register fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
5439 if (RetVT.SimpleTy != MVT::f32)
5440 return Register();
5441 if ((Subtarget->hasFPARMv8())) {
5442 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTSr, RC: &AArch64::FPR32RegClass, Op0);
5443 }
5444 return Register();
5445}
5446
5447Register fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
5448 if (RetVT.SimpleTy != MVT::f64)
5449 return Register();
5450 if ((Subtarget->hasFPARMv8())) {
5451 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTDr, RC: &AArch64::FPR64RegClass, Op0);
5452 }
5453 return Register();
5454}
5455
5456Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(MVT RetVT, Register Op0) {
5457 if (RetVT.SimpleTy != MVT::v4f16)
5458 return Register();
5459 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5460 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f16, RC: &AArch64::FPR64RegClass, Op0);
5461 }
5462 return Register();
5463}
5464
5465Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
5466 if (RetVT.SimpleTy != MVT::v8f16)
5467 return Register();
5468 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5469 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv8f16, RC: &AArch64::FPR128RegClass, Op0);
5470 }
5471 return Register();
5472}
5473
5474Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(MVT RetVT, Register Op0) {
5475 if (RetVT.SimpleTy != MVT::v2f32)
5476 return Register();
5477 if ((Subtarget->isNeonAvailable())) {
5478 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f32, RC: &AArch64::FPR64RegClass, Op0);
5479 }
5480 return Register();
5481}
5482
5483Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
5484 if (RetVT.SimpleTy != MVT::v4f32)
5485 return Register();
5486 if ((Subtarget->isNeonAvailable())) {
5487 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f32, RC: &AArch64::FPR128RegClass, Op0);
5488 }
5489 return Register();
5490}
5491
5492Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
5493 if (RetVT.SimpleTy != MVT::v2f64)
5494 return Register();
5495 if ((Subtarget->isNeonAvailable())) {
5496 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f64, RC: &AArch64::FPR128RegClass, Op0);
5497 }
5498 return Register();
5499}
5500
5501Register fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
5502 switch (VT.SimpleTy) {
5503 case MVT::f16: return fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(RetVT, Op0);
5504 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0);
5505 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0);
5506 case MVT::v4f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(RetVT, Op0);
5507 case MVT::v8f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(RetVT, Op0);
5508 case MVT::v2f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(RetVT, Op0);
5509 case MVT::v4f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(RetVT, Op0);
5510 case MVT::v2f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(RetVT, Op0);
5511 default: return Register();
5512 }
5513}
5514
5515// FastEmit functions for ISD::STRICT_FTRUNC.
5516
5517Register fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
5518 if (RetVT.SimpleTy != MVT::f16)
5519 return Register();
5520 if ((Subtarget->hasFullFP16())) {
5521 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZHr, RC: &AArch64::FPR16RegClass, Op0);
5522 }
5523 return Register();
5524}
5525
5526Register fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
5527 if (RetVT.SimpleTy != MVT::f32)
5528 return Register();
5529 if ((Subtarget->hasFPARMv8())) {
5530 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZSr, RC: &AArch64::FPR32RegClass, Op0);
5531 }
5532 return Register();
5533}
5534
5535Register fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
5536 if (RetVT.SimpleTy != MVT::f64)
5537 return Register();
5538 if ((Subtarget->hasFPARMv8())) {
5539 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZDr, RC: &AArch64::FPR64RegClass, Op0);
5540 }
5541 return Register();
5542}
5543
5544Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
5545 if (RetVT.SimpleTy != MVT::v4f16)
5546 return Register();
5547 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5548 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f16, RC: &AArch64::FPR64RegClass, Op0);
5549 }
5550 return Register();
5551}
5552
5553Register fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
5554 if (RetVT.SimpleTy != MVT::v8f16)
5555 return Register();
5556 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5557 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv8f16, RC: &AArch64::FPR128RegClass, Op0);
5558 }
5559 return Register();
5560}
5561
5562Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
5563 if (RetVT.SimpleTy != MVT::v2f32)
5564 return Register();
5565 if ((Subtarget->isNeonAvailable())) {
5566 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f32, RC: &AArch64::FPR64RegClass, Op0);
5567 }
5568 return Register();
5569}
5570
5571Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
5572 if (RetVT.SimpleTy != MVT::v4f32)
5573 return Register();
5574 if ((Subtarget->isNeonAvailable())) {
5575 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f32, RC: &AArch64::FPR128RegClass, Op0);
5576 }
5577 return Register();
5578}
5579
5580Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(MVT RetVT, Register Op0) {
5581 if (RetVT.SimpleTy != MVT::v2f64)
5582 return Register();
5583 if ((Subtarget->isNeonAvailable())) {
5584 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f64, RC: &AArch64::FPR128RegClass, Op0);
5585 }
5586 return Register();
5587}
5588
5589Register fastEmit_ISD_STRICT_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
5590 switch (VT.SimpleTy) {
5591 case MVT::f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(RetVT, Op0);
5592 case MVT::f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(RetVT, Op0);
5593 case MVT::f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(RetVT, Op0);
5594 case MVT::v4f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(RetVT, Op0);
5595 case MVT::v8f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(RetVT, Op0);
5596 case MVT::v2f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(RetVT, Op0);
5597 case MVT::v4f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(RetVT, Op0);
5598 case MVT::v2f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(RetVT, Op0);
5599 default: return Register();
5600 }
5601}
5602
5603// FastEmit functions for ISD::STRICT_LLROUND.
5604
5605Register fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5606 if (RetVT.SimpleTy != MVT::i64)
5607 return Register();
5608 if ((Subtarget->hasFullFP16())) {
5609 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
5610 }
5611 return Register();
5612}
5613
5614Register fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5615 if (RetVT.SimpleTy != MVT::i64)
5616 return Register();
5617 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
5618}
5619
5620Register fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5621 if (RetVT.SimpleTy != MVT::i64)
5622 return Register();
5623 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
5624}
5625
5626Register fastEmit_ISD_STRICT_LLROUND_r(MVT VT, MVT RetVT, Register Op0) {
5627 switch (VT.SimpleTy) {
5628 case MVT::f16: return fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(RetVT, Op0);
5629 case MVT::f32: return fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(RetVT, Op0);
5630 case MVT::f64: return fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(RetVT, Op0);
5631 default: return Register();
5632 }
5633}
5634
5635// FastEmit functions for ISD::STRICT_LROUND.
5636
5637Register fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(Register Op0) {
5638 if ((Subtarget->hasFullFP16())) {
5639 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWHr, RC: &AArch64::GPR32RegClass, Op0);
5640 }
5641 return Register();
5642}
5643
5644Register fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(Register Op0) {
5645 if ((Subtarget->hasFullFP16())) {
5646 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
5647 }
5648 return Register();
5649}
5650
5651Register fastEmit_ISD_STRICT_LROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5652switch (RetVT.SimpleTy) {
5653 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(Op0);
5654 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(Op0);
5655 default: return Register();
5656}
5657}
5658
5659Register fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(Register Op0) {
5660 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWSr, RC: &AArch64::GPR32RegClass, Op0);
5661}
5662
5663Register fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(Register Op0) {
5664 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
5665}
5666
5667Register fastEmit_ISD_STRICT_LROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5668switch (RetVT.SimpleTy) {
5669 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(Op0);
5670 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(Op0);
5671 default: return Register();
5672}
5673}
5674
5675Register fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(Register Op0) {
5676 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWDr, RC: &AArch64::GPR32RegClass, Op0);
5677}
5678
5679Register fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(Register Op0) {
5680 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
5681}
5682
5683Register fastEmit_ISD_STRICT_LROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5684switch (RetVT.SimpleTy) {
5685 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(Op0);
5686 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(Op0);
5687 default: return Register();
5688}
5689}
5690
5691Register fastEmit_ISD_STRICT_LROUND_r(MVT VT, MVT RetVT, Register Op0) {
5692 switch (VT.SimpleTy) {
5693 case MVT::f16: return fastEmit_ISD_STRICT_LROUND_MVT_f16_r(RetVT, Op0);
5694 case MVT::f32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_r(RetVT, Op0);
5695 case MVT::f64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_r(RetVT, Op0);
5696 default: return Register();
5697 }
5698}
5699
5700// FastEmit functions for ISD::STRICT_SINT_TO_FP.
5701
5702Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5703 if ((Subtarget->hasFPRCVT())) {
5704 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5705 }
5706 if ((Subtarget->hasFullFP16())) {
5707 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5708 }
5709 return Register();
5710}
5711
5712Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5713 if ((Subtarget->hasFPARMv8())) {
5714 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5715 }
5716 return Register();
5717}
5718
5719Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5720 if ((Subtarget->hasFPRCVT())) {
5721 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5722 }
5723 if ((Subtarget->hasFPARMv8())) {
5724 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5725 }
5726 return Register();
5727}
5728
5729Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5730switch (RetVT.SimpleTy) {
5731 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5732 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5733 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5734 default: return Register();
5735}
5736}
5737
5738Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5739 if ((Subtarget->hasFPRCVT())) {
5740 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5741 }
5742 if ((Subtarget->hasFullFP16())) {
5743 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5744 }
5745 return Register();
5746}
5747
5748Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5749 if ((Subtarget->hasFPRCVT())) {
5750 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5751 }
5752 if ((Subtarget->hasFPARMv8())) {
5753 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5754 }
5755 return Register();
5756}
5757
5758Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5759 if ((Subtarget->hasFPARMv8())) {
5760 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5761 }
5762 return Register();
5763}
5764
5765Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5766switch (RetVT.SimpleTy) {
5767 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5768 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5769 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5770 default: return Register();
5771}
5772}
5773
5774Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5775 if (RetVT.SimpleTy != MVT::v4f16)
5776 return Register();
5777 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5778 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5779 }
5780 return Register();
5781}
5782
5783Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5784 if (RetVT.SimpleTy != MVT::v8f16)
5785 return Register();
5786 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5787 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5788 }
5789 return Register();
5790}
5791
5792Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5793 if (RetVT.SimpleTy != MVT::v2f32)
5794 return Register();
5795 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5796 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5797 }
5798 return Register();
5799}
5800
5801Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5802 if (RetVT.SimpleTy != MVT::v4f32)
5803 return Register();
5804 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5805 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5806 }
5807 return Register();
5808}
5809
5810Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5811 if (RetVT.SimpleTy != MVT::v2f64)
5812 return Register();
5813 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5814 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5815 }
5816 return Register();
5817}
5818
5819Register fastEmit_ISD_STRICT_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
5820 switch (VT.SimpleTy) {
5821 case MVT::i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
5822 case MVT::i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
5823 case MVT::v4i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
5824 case MVT::v8i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
5825 case MVT::v2i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
5826 case MVT::v4i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
5827 case MVT::v2i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
5828 default: return Register();
5829 }
5830}
5831
5832// FastEmit functions for ISD::STRICT_UINT_TO_FP.
5833
5834Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5835 if ((Subtarget->hasFPRCVT())) {
5836 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5837 }
5838 if ((Subtarget->hasFullFP16())) {
5839 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5840 }
5841 return Register();
5842}
5843
5844Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5845 if ((Subtarget->hasFPARMv8())) {
5846 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5847 }
5848 return Register();
5849}
5850
5851Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5852 if ((Subtarget->hasFPRCVT())) {
5853 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5854 }
5855 if ((Subtarget->hasFPARMv8())) {
5856 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5857 }
5858 return Register();
5859}
5860
5861Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5862switch (RetVT.SimpleTy) {
5863 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5864 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5865 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5866 default: return Register();
5867}
5868}
5869
5870Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5871 if ((Subtarget->hasFPRCVT())) {
5872 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5873 }
5874 if ((Subtarget->hasFullFP16())) {
5875 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5876 }
5877 return Register();
5878}
5879
5880Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5881 if ((Subtarget->hasFPRCVT())) {
5882 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5883 }
5884 if ((Subtarget->hasFPARMv8())) {
5885 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5886 }
5887 return Register();
5888}
5889
5890Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5891 if ((Subtarget->hasFPARMv8())) {
5892 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5893 }
5894 return Register();
5895}
5896
5897Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5898switch (RetVT.SimpleTy) {
5899 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5900 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5901 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5902 default: return Register();
5903}
5904}
5905
5906Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5907 if (RetVT.SimpleTy != MVT::v4f16)
5908 return Register();
5909 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5910 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5911 }
5912 return Register();
5913}
5914
5915Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5916 if (RetVT.SimpleTy != MVT::v8f16)
5917 return Register();
5918 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5919 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5920 }
5921 return Register();
5922}
5923
5924Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5925 if (RetVT.SimpleTy != MVT::v2f32)
5926 return Register();
5927 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5928 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5929 }
5930 return Register();
5931}
5932
5933Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5934 if (RetVT.SimpleTy != MVT::v4f32)
5935 return Register();
5936 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5937 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5938 }
5939 return Register();
5940}
5941
5942Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5943 if (RetVT.SimpleTy != MVT::v2f64)
5944 return Register();
5945 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5946 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5947 }
5948 return Register();
5949}
5950
5951Register fastEmit_ISD_STRICT_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
5952 switch (VT.SimpleTy) {
5953 case MVT::i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(RetVT, Op0);
5954 case MVT::i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(RetVT, Op0);
5955 case MVT::v4i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
5956 case MVT::v8i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
5957 case MVT::v2i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
5958 case MVT::v4i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
5959 case MVT::v2i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
5960 default: return Register();
5961 }
5962}
5963
5964// FastEmit functions for ISD::TRUNCATE.
5965
5966Register fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, Register Op0) {
5967 if (RetVT.SimpleTy != MVT::i32)
5968 return Register();
5969 return fastEmitInst_extractsubreg(RetVT, Op0, Idx: AArch64::sub_32);
5970}
5971
5972Register fastEmit_ISD_TRUNCATE_MVT_v8i16_r(MVT RetVT, Register Op0) {
5973 if (RetVT.SimpleTy != MVT::v8i8)
5974 return Register();
5975 if ((Subtarget->isNeonAvailable())) {
5976 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5977 }
5978 return Register();
5979}
5980
5981Register fastEmit_ISD_TRUNCATE_MVT_v4i32_r(MVT RetVT, Register Op0) {
5982 if (RetVT.SimpleTy != MVT::v4i16)
5983 return Register();
5984 if ((Subtarget->isNeonAvailable())) {
5985 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5986 }
5987 return Register();
5988}
5989
5990Register fastEmit_ISD_TRUNCATE_MVT_v2i64_r(MVT RetVT, Register Op0) {
5991 if (RetVT.SimpleTy != MVT::v2i32)
5992 return Register();
5993 if ((Subtarget->isNeonAvailable())) {
5994 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5995 }
5996 return Register();
5997}
5998
5999Register fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, Register Op0) {
6000 switch (VT.SimpleTy) {
6001 case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0);
6002 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i16_r(RetVT, Op0);
6003 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_MVT_v4i32_r(RetVT, Op0);
6004 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_MVT_v2i64_r(RetVT, Op0);
6005 default: return Register();
6006 }
6007}
6008
6009// FastEmit functions for ISD::TRUNCATE_SSAT_S.
6010
6011Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v8i16_r(MVT RetVT, Register Op0) {
6012 if (RetVT.SimpleTy != MVT::v8i8)
6013 return Register();
6014 if ((Subtarget->isNeonAvailable())) {
6015 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
6016 }
6017 return Register();
6018}
6019
6020Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v4i32_r(MVT RetVT, Register Op0) {
6021 if (RetVT.SimpleTy != MVT::v4i16)
6022 return Register();
6023 if ((Subtarget->isNeonAvailable())) {
6024 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
6025 }
6026 return Register();
6027}
6028
6029Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v2i64_r(MVT RetVT, Register Op0) {
6030 if (RetVT.SimpleTy != MVT::v2i32)
6031 return Register();
6032 if ((Subtarget->isNeonAvailable())) {
6033 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
6034 }
6035 return Register();
6036}
6037
6038Register fastEmit_ISD_TRUNCATE_SSAT_S_r(MVT VT, MVT RetVT, Register Op0) {
6039 switch (VT.SimpleTy) {
6040 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v8i16_r(RetVT, Op0);
6041 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v4i32_r(RetVT, Op0);
6042 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v2i64_r(RetVT, Op0);
6043 default: return Register();
6044 }
6045}
6046
6047// FastEmit functions for ISD::TRUNCATE_SSAT_U.
6048
6049Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v8i16_r(MVT RetVT, Register Op0) {
6050 if (RetVT.SimpleTy != MVT::v8i8)
6051 return Register();
6052 if ((Subtarget->isNeonAvailable())) {
6053 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv8i8, RC: &AArch64::FPR64RegClass, Op0);
6054 }
6055 return Register();
6056}
6057
6058Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v4i32_r(MVT RetVT, Register Op0) {
6059 if (RetVT.SimpleTy != MVT::v4i16)
6060 return Register();
6061 if ((Subtarget->isNeonAvailable())) {
6062 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv4i16, RC: &AArch64::FPR64RegClass, Op0);
6063 }
6064 return Register();
6065}
6066
6067Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v2i64_r(MVT RetVT, Register Op0) {
6068 if (RetVT.SimpleTy != MVT::v2i32)
6069 return Register();
6070 if ((Subtarget->isNeonAvailable())) {
6071 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv2i32, RC: &AArch64::FPR64RegClass, Op0);
6072 }
6073 return Register();
6074}
6075
6076Register fastEmit_ISD_TRUNCATE_SSAT_U_r(MVT VT, MVT RetVT, Register Op0) {
6077 switch (VT.SimpleTy) {
6078 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v8i16_r(RetVT, Op0);
6079 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v4i32_r(RetVT, Op0);
6080 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v2i64_r(RetVT, Op0);
6081 default: return Register();
6082 }
6083}
6084
6085// FastEmit functions for ISD::TRUNCATE_USAT_U.
6086
6087Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v8i16_r(MVT RetVT, Register Op0) {
6088 if (RetVT.SimpleTy != MVT::v8i8)
6089 return Register();
6090 if ((Subtarget->isNeonAvailable())) {
6091 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
6092 }
6093 return Register();
6094}
6095
6096Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v4i32_r(MVT RetVT, Register Op0) {
6097 if (RetVT.SimpleTy != MVT::v4i16)
6098 return Register();
6099 if ((Subtarget->isNeonAvailable())) {
6100 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
6101 }
6102 return Register();
6103}
6104
6105Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v2i64_r(MVT RetVT, Register Op0) {
6106 if (RetVT.SimpleTy != MVT::v2i32)
6107 return Register();
6108 if ((Subtarget->isNeonAvailable())) {
6109 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
6110 }
6111 return Register();
6112}
6113
6114Register fastEmit_ISD_TRUNCATE_USAT_U_r(MVT VT, MVT RetVT, Register Op0) {
6115 switch (VT.SimpleTy) {
6116 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v8i16_r(RetVT, Op0);
6117 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v4i32_r(RetVT, Op0);
6118 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v2i64_r(RetVT, Op0);
6119 default: return Register();
6120 }
6121}
6122
6123// FastEmit functions for ISD::UINT_TO_FP.
6124
6125Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
6126 if ((Subtarget->hasFPRCVT())) {
6127 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
6128 }
6129 if ((Subtarget->hasFullFP16())) {
6130 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
6131 }
6132 return Register();
6133}
6134
6135Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
6136 if ((Subtarget->hasFPARMv8())) {
6137 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
6138 }
6139 return Register();
6140}
6141
6142Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
6143 if ((Subtarget->hasFPRCVT())) {
6144 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
6145 }
6146 if ((Subtarget->hasFPARMv8())) {
6147 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
6148 }
6149 return Register();
6150}
6151
6152Register fastEmit_ISD_UINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
6153switch (RetVT.SimpleTy) {
6154 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
6155 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
6156 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
6157 default: return Register();
6158}
6159}
6160
6161Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
6162 if ((Subtarget->hasFPRCVT())) {
6163 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
6164 }
6165 if ((Subtarget->hasFullFP16())) {
6166 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
6167 }
6168 return Register();
6169}
6170
6171Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
6172 if ((Subtarget->hasFPRCVT())) {
6173 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
6174 }
6175 if ((Subtarget->hasFPARMv8())) {
6176 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
6177 }
6178 return Register();
6179}
6180
6181Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
6182 if ((Subtarget->hasFPARMv8())) {
6183 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
6184 }
6185 return Register();
6186}
6187
6188Register fastEmit_ISD_UINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
6189switch (RetVT.SimpleTy) {
6190 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
6191 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
6192 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
6193 default: return Register();
6194}
6195}
6196
6197Register fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
6198 if (RetVT.SimpleTy != MVT::v4f16)
6199 return Register();
6200 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
6201 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
6202 }
6203 return Register();
6204}
6205
6206Register fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
6207 if (RetVT.SimpleTy != MVT::v8f16)
6208 return Register();
6209 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
6210 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
6211 }
6212 return Register();
6213}
6214
6215Register fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
6216 if (RetVT.SimpleTy != MVT::v2f32)
6217 return Register();
6218 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
6219 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
6220 }
6221 return Register();
6222}
6223
6224Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
6225 if (RetVT.SimpleTy != MVT::v4f32)
6226 return Register();
6227 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
6228 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
6229 }
6230 return Register();
6231}
6232
6233Register fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
6234 if (RetVT.SimpleTy != MVT::v2f64)
6235 return Register();
6236 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
6237 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
6238 }
6239 return Register();
6240}
6241
6242Register fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
6243 switch (VT.SimpleTy) {
6244 case MVT::i32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_r(RetVT, Op0);
6245 case MVT::i64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_r(RetVT, Op0);
6246 case MVT::v4i16: return fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
6247 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
6248 case MVT::v2i32: return fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
6249 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
6250 case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
6251 default: return Register();
6252 }
6253}
6254
6255// FastEmit functions for ISD::VECREDUCE_ADD.
6256
6257Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i8_r(MVT RetVT, Register Op0) {
6258 if (RetVT.SimpleTy != MVT::i8)
6259 return Register();
6260 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6261}
6262
6263Register fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(MVT RetVT, Register Op0) {
6264 if (RetVT.SimpleTy != MVT::i8)
6265 return Register();
6266 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6267}
6268
6269Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i16_r(MVT RetVT, Register Op0) {
6270 if (RetVT.SimpleTy != MVT::i16)
6271 return Register();
6272 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6273}
6274
6275Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(MVT RetVT, Register Op0) {
6276 if (RetVT.SimpleTy != MVT::i16)
6277 return Register();
6278 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6279}
6280
6281Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(MVT RetVT, Register Op0) {
6282 if (RetVT.SimpleTy != MVT::i32)
6283 return Register();
6284 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6285}
6286
6287Register fastEmit_ISD_VECREDUCE_ADD_MVT_v2i64_r(MVT RetVT, Register Op0) {
6288 if (RetVT.SimpleTy != MVT::i64)
6289 return Register();
6290 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6291}
6292
6293Register fastEmit_ISD_VECREDUCE_ADD_r(MVT VT, MVT RetVT, Register Op0) {
6294 switch (VT.SimpleTy) {
6295 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i8_r(RetVT, Op0);
6296 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(RetVT, Op0);
6297 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i16_r(RetVT, Op0);
6298 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(RetVT, Op0);
6299 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(RetVT, Op0);
6300 case MVT::v2i64: return fastEmit_ISD_VECREDUCE_ADD_MVT_v2i64_r(RetVT, Op0);
6301 default: return Register();
6302 }
6303}
6304
6305// FastEmit functions for ISD::VECREDUCE_FADD.
6306
6307Register fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(MVT RetVT, Register Op0) {
6308 if (RetVT.SimpleTy != MVT::f32)
6309 return Register();
6310 return fastEmitInst_r(MachineInstOpcode: AArch64::FADDPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6311}
6312
6313Register fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(MVT RetVT, Register Op0) {
6314 if (RetVT.SimpleTy != MVT::f64)
6315 return Register();
6316 return fastEmitInst_r(MachineInstOpcode: AArch64::FADDPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6317}
6318
6319Register fastEmit_ISD_VECREDUCE_FADD_r(MVT VT, MVT RetVT, Register Op0) {
6320 switch (VT.SimpleTy) {
6321 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(RetVT, Op0);
6322 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(RetVT, Op0);
6323 default: return Register();
6324 }
6325}
6326
6327// FastEmit functions for ISD::VECREDUCE_FMAX.
6328
6329Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f16_r(MVT RetVT, Register Op0) {
6330 if (RetVT.SimpleTy != MVT::f16)
6331 return Register();
6332 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6333 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6334 }
6335 return Register();
6336}
6337
6338Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v8f16_r(MVT RetVT, Register Op0) {
6339 if (RetVT.SimpleTy != MVT::f16)
6340 return Register();
6341 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6342 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6343 }
6344 return Register();
6345}
6346
6347Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f32_r(MVT RetVT, Register Op0) {
6348 if (RetVT.SimpleTy != MVT::f32)
6349 return Register();
6350 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6351}
6352
6353Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f32_r(MVT RetVT, Register Op0) {
6354 if (RetVT.SimpleTy != MVT::f32)
6355 return Register();
6356 if ((Subtarget->isNeonAvailable())) {
6357 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6358 }
6359 return Register();
6360}
6361
6362Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f64_r(MVT RetVT, Register Op0) {
6363 if (RetVT.SimpleTy != MVT::f64)
6364 return Register();
6365 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6366}
6367
6368Register fastEmit_ISD_VECREDUCE_FMAX_r(MVT VT, MVT RetVT, Register Op0) {
6369 switch (VT.SimpleTy) {
6370 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f16_r(RetVT, Op0);
6371 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v8f16_r(RetVT, Op0);
6372 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f32_r(RetVT, Op0);
6373 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f32_r(RetVT, Op0);
6374 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f64_r(RetVT, Op0);
6375 default: return Register();
6376 }
6377}
6378
6379// FastEmit functions for ISD::VECREDUCE_FMAXIMUM.
6380
6381Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f16_r(MVT RetVT, Register Op0) {
6382 if (RetVT.SimpleTy != MVT::f16)
6383 return Register();
6384 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6385 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6386 }
6387 return Register();
6388}
6389
6390Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v8f16_r(MVT RetVT, Register Op0) {
6391 if (RetVT.SimpleTy != MVT::f16)
6392 return Register();
6393 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6394 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6395 }
6396 return Register();
6397}
6398
6399Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f32_r(MVT RetVT, Register Op0) {
6400 if (RetVT.SimpleTy != MVT::f32)
6401 return Register();
6402 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6403}
6404
6405Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f32_r(MVT RetVT, Register Op0) {
6406 if (RetVT.SimpleTy != MVT::f32)
6407 return Register();
6408 if ((Subtarget->isNeonAvailable())) {
6409 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6410 }
6411 return Register();
6412}
6413
6414Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f64_r(MVT RetVT, Register Op0) {
6415 if (RetVT.SimpleTy != MVT::f64)
6416 return Register();
6417 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6418}
6419
6420Register fastEmit_ISD_VECREDUCE_FMAXIMUM_r(MVT VT, MVT RetVT, Register Op0) {
6421 switch (VT.SimpleTy) {
6422 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f16_r(RetVT, Op0);
6423 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v8f16_r(RetVT, Op0);
6424 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f32_r(RetVT, Op0);
6425 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f32_r(RetVT, Op0);
6426 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f64_r(RetVT, Op0);
6427 default: return Register();
6428 }
6429}
6430
6431// FastEmit functions for ISD::VECREDUCE_FMIN.
6432
6433Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f16_r(MVT RetVT, Register Op0) {
6434 if (RetVT.SimpleTy != MVT::f16)
6435 return Register();
6436 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6437 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6438 }
6439 return Register();
6440}
6441
6442Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v8f16_r(MVT RetVT, Register Op0) {
6443 if (RetVT.SimpleTy != MVT::f16)
6444 return Register();
6445 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6446 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6447 }
6448 return Register();
6449}
6450
6451Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f32_r(MVT RetVT, Register Op0) {
6452 if (RetVT.SimpleTy != MVT::f32)
6453 return Register();
6454 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6455}
6456
6457Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f32_r(MVT RetVT, Register Op0) {
6458 if (RetVT.SimpleTy != MVT::f32)
6459 return Register();
6460 if ((Subtarget->isNeonAvailable())) {
6461 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6462 }
6463 return Register();
6464}
6465
6466Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f64_r(MVT RetVT, Register Op0) {
6467 if (RetVT.SimpleTy != MVT::f64)
6468 return Register();
6469 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6470}
6471
6472Register fastEmit_ISD_VECREDUCE_FMIN_r(MVT VT, MVT RetVT, Register Op0) {
6473 switch (VT.SimpleTy) {
6474 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f16_r(RetVT, Op0);
6475 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v8f16_r(RetVT, Op0);
6476 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f32_r(RetVT, Op0);
6477 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f32_r(RetVT, Op0);
6478 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f64_r(RetVT, Op0);
6479 default: return Register();
6480 }
6481}
6482
6483// FastEmit functions for ISD::VECREDUCE_FMINIMUM.
6484
6485Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f16_r(MVT RetVT, Register Op0) {
6486 if (RetVT.SimpleTy != MVT::f16)
6487 return Register();
6488 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6489 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6490 }
6491 return Register();
6492}
6493
6494Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v8f16_r(MVT RetVT, Register Op0) {
6495 if (RetVT.SimpleTy != MVT::f16)
6496 return Register();
6497 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6498 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6499 }
6500 return Register();
6501}
6502
6503Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f32_r(MVT RetVT, Register Op0) {
6504 if (RetVT.SimpleTy != MVT::f32)
6505 return Register();
6506 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6507}
6508
6509Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f32_r(MVT RetVT, Register Op0) {
6510 if (RetVT.SimpleTy != MVT::f32)
6511 return Register();
6512 if ((Subtarget->isNeonAvailable())) {
6513 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6514 }
6515 return Register();
6516}
6517
6518Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f64_r(MVT RetVT, Register Op0) {
6519 if (RetVT.SimpleTy != MVT::f64)
6520 return Register();
6521 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6522}
6523
6524Register fastEmit_ISD_VECREDUCE_FMINIMUM_r(MVT VT, MVT RetVT, Register Op0) {
6525 switch (VT.SimpleTy) {
6526 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f16_r(RetVT, Op0);
6527 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v8f16_r(RetVT, Op0);
6528 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f32_r(RetVT, Op0);
6529 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f32_r(RetVT, Op0);
6530 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f64_r(RetVT, Op0);
6531 default: return Register();
6532 }
6533}
6534
6535// FastEmit functions for ISD::VECREDUCE_SMAX.
6536
6537Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i8_r(MVT RetVT, Register Op0) {
6538 if (RetVT.SimpleTy != MVT::i8)
6539 return Register();
6540 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6541}
6542
6543Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v16i8_r(MVT RetVT, Register Op0) {
6544 if (RetVT.SimpleTy != MVT::i8)
6545 return Register();
6546 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6547}
6548
6549Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i16_r(MVT RetVT, Register Op0) {
6550 if (RetVT.SimpleTy != MVT::i16)
6551 return Register();
6552 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6553}
6554
6555Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i16_r(MVT RetVT, Register Op0) {
6556 if (RetVT.SimpleTy != MVT::i16)
6557 return Register();
6558 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6559}
6560
6561Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i32_r(MVT RetVT, Register Op0) {
6562 if (RetVT.SimpleTy != MVT::i32)
6563 return Register();
6564 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6565}
6566
6567Register fastEmit_ISD_VECREDUCE_SMAX_r(MVT VT, MVT RetVT, Register Op0) {
6568 switch (VT.SimpleTy) {
6569 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i8_r(RetVT, Op0);
6570 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v16i8_r(RetVT, Op0);
6571 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i16_r(RetVT, Op0);
6572 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i16_r(RetVT, Op0);
6573 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i32_r(RetVT, Op0);
6574 default: return Register();
6575 }
6576}
6577
6578// FastEmit functions for ISD::VECREDUCE_SMIN.
6579
6580Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i8_r(MVT RetVT, Register Op0) {
6581 if (RetVT.SimpleTy != MVT::i8)
6582 return Register();
6583 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6584}
6585
6586Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v16i8_r(MVT RetVT, Register Op0) {
6587 if (RetVT.SimpleTy != MVT::i8)
6588 return Register();
6589 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6590}
6591
6592Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i16_r(MVT RetVT, Register Op0) {
6593 if (RetVT.SimpleTy != MVT::i16)
6594 return Register();
6595 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6596}
6597
6598Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i16_r(MVT RetVT, Register Op0) {
6599 if (RetVT.SimpleTy != MVT::i16)
6600 return Register();
6601 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6602}
6603
6604Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i32_r(MVT RetVT, Register Op0) {
6605 if (RetVT.SimpleTy != MVT::i32)
6606 return Register();
6607 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6608}
6609
6610Register fastEmit_ISD_VECREDUCE_SMIN_r(MVT VT, MVT RetVT, Register Op0) {
6611 switch (VT.SimpleTy) {
6612 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i8_r(RetVT, Op0);
6613 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v16i8_r(RetVT, Op0);
6614 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i16_r(RetVT, Op0);
6615 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i16_r(RetVT, Op0);
6616 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i32_r(RetVT, Op0);
6617 default: return Register();
6618 }
6619}
6620
6621// FastEmit functions for ISD::VECREDUCE_UMAX.
6622
6623Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i8_r(MVT RetVT, Register Op0) {
6624 if (RetVT.SimpleTy != MVT::i8)
6625 return Register();
6626 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6627}
6628
6629Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v16i8_r(MVT RetVT, Register Op0) {
6630 if (RetVT.SimpleTy != MVT::i8)
6631 return Register();
6632 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6633}
6634
6635Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i16_r(MVT RetVT, Register Op0) {
6636 if (RetVT.SimpleTy != MVT::i16)
6637 return Register();
6638 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6639}
6640
6641Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i16_r(MVT RetVT, Register Op0) {
6642 if (RetVT.SimpleTy != MVT::i16)
6643 return Register();
6644 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6645}
6646
6647Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i32_r(MVT RetVT, Register Op0) {
6648 if (RetVT.SimpleTy != MVT::i32)
6649 return Register();
6650 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6651}
6652
6653Register fastEmit_ISD_VECREDUCE_UMAX_r(MVT VT, MVT RetVT, Register Op0) {
6654 switch (VT.SimpleTy) {
6655 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i8_r(RetVT, Op0);
6656 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v16i8_r(RetVT, Op0);
6657 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i16_r(RetVT, Op0);
6658 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i16_r(RetVT, Op0);
6659 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i32_r(RetVT, Op0);
6660 default: return Register();
6661 }
6662}
6663
6664// FastEmit functions for ISD::VECREDUCE_UMIN.
6665
6666Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i8_r(MVT RetVT, Register Op0) {
6667 if (RetVT.SimpleTy != MVT::i8)
6668 return Register();
6669 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6670}
6671
6672Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v16i8_r(MVT RetVT, Register Op0) {
6673 if (RetVT.SimpleTy != MVT::i8)
6674 return Register();
6675 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6676}
6677
6678Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i16_r(MVT RetVT, Register Op0) {
6679 if (RetVT.SimpleTy != MVT::i16)
6680 return Register();
6681 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6682}
6683
6684Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i16_r(MVT RetVT, Register Op0) {
6685 if (RetVT.SimpleTy != MVT::i16)
6686 return Register();
6687 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6688}
6689
6690Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i32_r(MVT RetVT, Register Op0) {
6691 if (RetVT.SimpleTy != MVT::i32)
6692 return Register();
6693 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6694}
6695
6696Register fastEmit_ISD_VECREDUCE_UMIN_r(MVT VT, MVT RetVT, Register Op0) {
6697 switch (VT.SimpleTy) {
6698 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i8_r(RetVT, Op0);
6699 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v16i8_r(RetVT, Op0);
6700 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i16_r(RetVT, Op0);
6701 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i16_r(RetVT, Op0);
6702 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i32_r(RetVT, Op0);
6703 default: return Register();
6704 }
6705}
6706
6707// FastEmit functions for ISD::VECTOR_REVERSE.
6708
6709Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(MVT RetVT, Register Op0) {
6710 if (RetVT.SimpleTy != MVT::nxv2i1)
6711 return Register();
6712 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6713 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_D, RC: &AArch64::PPRRegClass, Op0);
6714 }
6715 return Register();
6716}
6717
6718Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(MVT RetVT, Register Op0) {
6719 if (RetVT.SimpleTy != MVT::nxv4i1)
6720 return Register();
6721 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6722 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_S, RC: &AArch64::PPRRegClass, Op0);
6723 }
6724 return Register();
6725}
6726
6727Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(MVT RetVT, Register Op0) {
6728 if (RetVT.SimpleTy != MVT::nxv8i1)
6729 return Register();
6730 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6731 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_H, RC: &AArch64::PPRRegClass, Op0);
6732 }
6733 return Register();
6734}
6735
6736Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(MVT RetVT, Register Op0) {
6737 if (RetVT.SimpleTy != MVT::nxv16i1)
6738 return Register();
6739 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6740 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_B, RC: &AArch64::PPRRegClass, Op0);
6741 }
6742 return Register();
6743}
6744
6745Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
6746 if (RetVT.SimpleTy != MVT::nxv16i8)
6747 return Register();
6748 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6749 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_B, RC: &AArch64::ZPRRegClass, Op0);
6750 }
6751 return Register();
6752}
6753
6754Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
6755 if (RetVT.SimpleTy != MVT::nxv8i16)
6756 return Register();
6757 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6758 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6759 }
6760 return Register();
6761}
6762
6763Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
6764 if (RetVT.SimpleTy != MVT::nxv4i32)
6765 return Register();
6766 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6767 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6768 }
6769 return Register();
6770}
6771
6772Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(MVT RetVT, Register Op0) {
6773 if (RetVT.SimpleTy != MVT::nxv2i64)
6774 return Register();
6775 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6776 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6777 }
6778 return Register();
6779}
6780
6781Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(MVT RetVT, Register Op0) {
6782 if (RetVT.SimpleTy != MVT::nxv2f16)
6783 return Register();
6784 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6785 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6786 }
6787 return Register();
6788}
6789
6790Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(MVT RetVT, Register Op0) {
6791 if (RetVT.SimpleTy != MVT::nxv4f16)
6792 return Register();
6793 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6794 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6795 }
6796 return Register();
6797}
6798
6799Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
6800 if (RetVT.SimpleTy != MVT::nxv8f16)
6801 return Register();
6802 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6803 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6804 }
6805 return Register();
6806}
6807
6808Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(MVT RetVT, Register Op0) {
6809 if (RetVT.SimpleTy != MVT::nxv2bf16)
6810 return Register();
6811 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6812 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6813 }
6814 return Register();
6815}
6816
6817Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(MVT RetVT, Register Op0) {
6818 if (RetVT.SimpleTy != MVT::nxv4bf16)
6819 return Register();
6820 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6821 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6822 }
6823 return Register();
6824}
6825
6826Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(MVT RetVT, Register Op0) {
6827 if (RetVT.SimpleTy != MVT::nxv8bf16)
6828 return Register();
6829 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6830 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6831 }
6832 return Register();
6833}
6834
6835Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(MVT RetVT, Register Op0) {
6836 if (RetVT.SimpleTy != MVT::nxv2f32)
6837 return Register();
6838 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6839 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6840 }
6841 return Register();
6842}
6843
6844Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
6845 if (RetVT.SimpleTy != MVT::nxv4f32)
6846 return Register();
6847 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6848 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6849 }
6850 return Register();
6851}
6852
6853Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
6854 if (RetVT.SimpleTy != MVT::nxv2f64)
6855 return Register();
6856 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6857 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6858 }
6859 return Register();
6860}
6861
6862Register fastEmit_ISD_VECTOR_REVERSE_r(MVT VT, MVT RetVT, Register Op0) {
6863 switch (VT.SimpleTy) {
6864 case MVT::nxv2i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(RetVT, Op0);
6865 case MVT::nxv4i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(RetVT, Op0);
6866 case MVT::nxv8i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(RetVT, Op0);
6867 case MVT::nxv16i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(RetVT, Op0);
6868 case MVT::nxv16i8: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(RetVT, Op0);
6869 case MVT::nxv8i16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(RetVT, Op0);
6870 case MVT::nxv4i32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(RetVT, Op0);
6871 case MVT::nxv2i64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(RetVT, Op0);
6872 case MVT::nxv2f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(RetVT, Op0);
6873 case MVT::nxv4f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(RetVT, Op0);
6874 case MVT::nxv8f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(RetVT, Op0);
6875 case MVT::nxv2bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(RetVT, Op0);
6876 case MVT::nxv4bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(RetVT, Op0);
6877 case MVT::nxv8bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(RetVT, Op0);
6878 case MVT::nxv2f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(RetVT, Op0);
6879 case MVT::nxv4f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(RetVT, Op0);
6880 case MVT::nxv2f64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(RetVT, Op0);
6881 default: return Register();
6882 }
6883}
6884
6885// Top-level FastEmit function.
6886
6887Register fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, Register Op0) override {
6888 switch (Opcode) {
6889 case AArch64ISD::ALLOCATE_ZA_BUFFER: return fastEmit_AArch64ISD_ALLOCATE_ZA_BUFFER_r(VT, RetVT, Op0);
6890 case AArch64ISD::ALLOC_SME_SAVE_BUFFER: return fastEmit_AArch64ISD_ALLOC_SME_SAVE_BUFFER_r(VT, RetVT, Op0);
6891 case AArch64ISD::CALL: return fastEmit_AArch64ISD_CALL_r(VT, RetVT, Op0);
6892 case AArch64ISD::COALESCER_BARRIER: return fastEmit_AArch64ISD_COALESCER_BARRIER_r(VT, RetVT, Op0);
6893 case AArch64ISD::DUP: return fastEmit_AArch64ISD_DUP_r(VT, RetVT, Op0);
6894 case AArch64ISD::FCVTXN: return fastEmit_AArch64ISD_FCVTXN_r(VT, RetVT, Op0);
6895 case AArch64ISD::FRECPE: return fastEmit_AArch64ISD_FRECPE_r(VT, RetVT, Op0);
6896 case AArch64ISD::FRSQRTE: return fastEmit_AArch64ISD_FRSQRTE_r(VT, RetVT, Op0);
6897 case AArch64ISD::PROBED_ALLOCA: return fastEmit_AArch64ISD_PROBED_ALLOCA_r(VT, RetVT, Op0);
6898 case AArch64ISD::REV16: return fastEmit_AArch64ISD_REV16_r(VT, RetVT, Op0);
6899 case AArch64ISD::REV32: return fastEmit_AArch64ISD_REV32_r(VT, RetVT, Op0);
6900 case AArch64ISD::REV64: return fastEmit_AArch64ISD_REV64_r(VT, RetVT, Op0);
6901 case AArch64ISD::SADDLP: return fastEmit_AArch64ISD_SADDLP_r(VT, RetVT, Op0);
6902 case AArch64ISD::SITOF: return fastEmit_AArch64ISD_SITOF_r(VT, RetVT, Op0);
6903 case AArch64ISD::SQABS: return fastEmit_AArch64ISD_SQABS_r(VT, RetVT, Op0);
6904 case AArch64ISD::SQNEG: return fastEmit_AArch64ISD_SQNEG_r(VT, RetVT, Op0);
6905 case AArch64ISD::SUNPKHI: return fastEmit_AArch64ISD_SUNPKHI_r(VT, RetVT, Op0);
6906 case AArch64ISD::SUNPKLO: return fastEmit_AArch64ISD_SUNPKLO_r(VT, RetVT, Op0);
6907 case AArch64ISD::UADDLP: return fastEmit_AArch64ISD_UADDLP_r(VT, RetVT, Op0);
6908 case AArch64ISD::UITOF: return fastEmit_AArch64ISD_UITOF_r(VT, RetVT, Op0);
6909 case AArch64ISD::UUNPKHI: return fastEmit_AArch64ISD_UUNPKHI_r(VT, RetVT, Op0);
6910 case AArch64ISD::UUNPKLO: return fastEmit_AArch64ISD_UUNPKLO_r(VT, RetVT, Op0);
6911 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
6912 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
6913 case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0);
6914 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0);
6915 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
6916 case ISD::CTLS: return fastEmit_ISD_CTLS_r(VT, RetVT, Op0);
6917 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
6918 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
6919 case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0);
6920 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
6921 case ISD::FCEIL: return fastEmit_ISD_FCEIL_r(VT, RetVT, Op0);
6922 case ISD::FFLOOR: return fastEmit_ISD_FFLOOR_r(VT, RetVT, Op0);
6923 case ISD::FNEARBYINT: return fastEmit_ISD_FNEARBYINT_r(VT, RetVT, Op0);
6924 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
6925 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
6926 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
6927 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
6928 case ISD::FP_TO_SINT_SAT: return fastEmit_ISD_FP_TO_SINT_SAT_r(VT, RetVT, Op0);
6929 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
6930 case ISD::FP_TO_UINT_SAT: return fastEmit_ISD_FP_TO_UINT_SAT_r(VT, RetVT, Op0);
6931 case ISD::FRINT: return fastEmit_ISD_FRINT_r(VT, RetVT, Op0);
6932 case ISD::FROUND: return fastEmit_ISD_FROUND_r(VT, RetVT, Op0);
6933 case ISD::FROUNDEVEN: return fastEmit_ISD_FROUNDEVEN_r(VT, RetVT, Op0);
6934 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
6935 case ISD::FTRUNC: return fastEmit_ISD_FTRUNC_r(VT, RetVT, Op0);
6936 case ISD::LLROUND: return fastEmit_ISD_LLROUND_r(VT, RetVT, Op0);
6937 case ISD::LROUND: return fastEmit_ISD_LROUND_r(VT, RetVT, Op0);
6938 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
6939 case ISD::SPLAT_VECTOR: return fastEmit_ISD_SPLAT_VECTOR_r(VT, RetVT, Op0);
6940 case ISD::STRICT_FCEIL: return fastEmit_ISD_STRICT_FCEIL_r(VT, RetVT, Op0);
6941 case ISD::STRICT_FFLOOR: return fastEmit_ISD_STRICT_FFLOOR_r(VT, RetVT, Op0);
6942 case ISD::STRICT_FNEARBYINT: return fastEmit_ISD_STRICT_FNEARBYINT_r(VT, RetVT, Op0);
6943 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0);
6944 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0);
6945 case ISD::STRICT_FP_TO_SINT: return fastEmit_ISD_STRICT_FP_TO_SINT_r(VT, RetVT, Op0);
6946 case ISD::STRICT_FP_TO_UINT: return fastEmit_ISD_STRICT_FP_TO_UINT_r(VT, RetVT, Op0);
6947 case ISD::STRICT_FRINT: return fastEmit_ISD_STRICT_FRINT_r(VT, RetVT, Op0);
6948 case ISD::STRICT_FROUND: return fastEmit_ISD_STRICT_FROUND_r(VT, RetVT, Op0);
6949 case ISD::STRICT_FROUNDEVEN: return fastEmit_ISD_STRICT_FROUNDEVEN_r(VT, RetVT, Op0);
6950 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0);
6951 case ISD::STRICT_FTRUNC: return fastEmit_ISD_STRICT_FTRUNC_r(VT, RetVT, Op0);
6952 case ISD::STRICT_LLROUND: return fastEmit_ISD_STRICT_LLROUND_r(VT, RetVT, Op0);
6953 case ISD::STRICT_LROUND: return fastEmit_ISD_STRICT_LROUND_r(VT, RetVT, Op0);
6954 case ISD::STRICT_SINT_TO_FP: return fastEmit_ISD_STRICT_SINT_TO_FP_r(VT, RetVT, Op0);
6955 case ISD::STRICT_UINT_TO_FP: return fastEmit_ISD_STRICT_UINT_TO_FP_r(VT, RetVT, Op0);
6956 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
6957 case ISD::TRUNCATE_SSAT_S: return fastEmit_ISD_TRUNCATE_SSAT_S_r(VT, RetVT, Op0);
6958 case ISD::TRUNCATE_SSAT_U: return fastEmit_ISD_TRUNCATE_SSAT_U_r(VT, RetVT, Op0);
6959 case ISD::TRUNCATE_USAT_U: return fastEmit_ISD_TRUNCATE_USAT_U_r(VT, RetVT, Op0);
6960 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
6961 case ISD::VECREDUCE_ADD: return fastEmit_ISD_VECREDUCE_ADD_r(VT, RetVT, Op0);
6962 case ISD::VECREDUCE_FADD: return fastEmit_ISD_VECREDUCE_FADD_r(VT, RetVT, Op0);
6963 case ISD::VECREDUCE_FMAX: return fastEmit_ISD_VECREDUCE_FMAX_r(VT, RetVT, Op0);
6964 case ISD::VECREDUCE_FMAXIMUM: return fastEmit_ISD_VECREDUCE_FMAXIMUM_r(VT, RetVT, Op0);
6965 case ISD::VECREDUCE_FMIN: return fastEmit_ISD_VECREDUCE_FMIN_r(VT, RetVT, Op0);
6966 case ISD::VECREDUCE_FMINIMUM: return fastEmit_ISD_VECREDUCE_FMINIMUM_r(VT, RetVT, Op0);
6967 case ISD::VECREDUCE_SMAX: return fastEmit_ISD_VECREDUCE_SMAX_r(VT, RetVT, Op0);
6968 case ISD::VECREDUCE_SMIN: return fastEmit_ISD_VECREDUCE_SMIN_r(VT, RetVT, Op0);
6969 case ISD::VECREDUCE_UMAX: return fastEmit_ISD_VECREDUCE_UMAX_r(VT, RetVT, Op0);
6970 case ISD::VECREDUCE_UMIN: return fastEmit_ISD_VECREDUCE_UMIN_r(VT, RetVT, Op0);
6971 case ISD::VECTOR_REVERSE: return fastEmit_ISD_VECTOR_REVERSE_r(VT, RetVT, Op0);
6972 default: return Register();
6973 }
6974}
6975
6976// FastEmit functions for AArch64ISD::ADDP.
6977
6978Register fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6979 if (RetVT.SimpleTy != MVT::v8i8)
6980 return Register();
6981 if ((Subtarget->isNeonAvailable())) {
6982 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
6983 }
6984 return Register();
6985}
6986
6987Register fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6988 if (RetVT.SimpleTy != MVT::v16i8)
6989 return Register();
6990 if ((Subtarget->isNeonAvailable())) {
6991 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
6992 }
6993 return Register();
6994}
6995
6996Register fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6997 if (RetVT.SimpleTy != MVT::v4i16)
6998 return Register();
6999 if ((Subtarget->isNeonAvailable())) {
7000 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7001 }
7002 return Register();
7003}
7004
7005Register fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7006 if (RetVT.SimpleTy != MVT::v8i16)
7007 return Register();
7008 if ((Subtarget->isNeonAvailable())) {
7009 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7010 }
7011 return Register();
7012}
7013
7014Register fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7015 if (RetVT.SimpleTy != MVT::v2i32)
7016 return Register();
7017 if ((Subtarget->isNeonAvailable())) {
7018 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7019 }
7020 return Register();
7021}
7022
7023Register fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7024 if (RetVT.SimpleTy != MVT::v4i32)
7025 return Register();
7026 if ((Subtarget->isNeonAvailable())) {
7027 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7028 }
7029 return Register();
7030}
7031
7032Register fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7033 if (RetVT.SimpleTy != MVT::v2i64)
7034 return Register();
7035 if ((Subtarget->isNeonAvailable())) {
7036 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7037 }
7038 return Register();
7039}
7040
7041Register fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7042 if (RetVT.SimpleTy != MVT::v4f16)
7043 return Register();
7044 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7045 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7046 }
7047 return Register();
7048}
7049
7050Register fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7051 if (RetVT.SimpleTy != MVT::v8f16)
7052 return Register();
7053 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7054 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7055 }
7056 return Register();
7057}
7058
7059Register fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7060 if (RetVT.SimpleTy != MVT::v2f32)
7061 return Register();
7062 if ((Subtarget->isNeonAvailable())) {
7063 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7064 }
7065 return Register();
7066}
7067
7068Register fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7069 if (RetVT.SimpleTy != MVT::v4f32)
7070 return Register();
7071 if ((Subtarget->isNeonAvailable())) {
7072 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7073 }
7074 return Register();
7075}
7076
7077Register fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7078 if (RetVT.SimpleTy != MVT::v2f64)
7079 return Register();
7080 if ((Subtarget->isNeonAvailable())) {
7081 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7082 }
7083 return Register();
7084}
7085
7086Register fastEmit_AArch64ISD_ADDP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7087 switch (VT.SimpleTy) {
7088 case MVT::v8i8: return fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(RetVT, Op0, Op1);
7089 case MVT::v16i8: return fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(RetVT, Op0, Op1);
7090 case MVT::v4i16: return fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(RetVT, Op0, Op1);
7091 case MVT::v8i16: return fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(RetVT, Op0, Op1);
7092 case MVT::v2i32: return fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(RetVT, Op0, Op1);
7093 case MVT::v4i32: return fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(RetVT, Op0, Op1);
7094 case MVT::v2i64: return fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(RetVT, Op0, Op1);
7095 case MVT::v4f16: return fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(RetVT, Op0, Op1);
7096 case MVT::v8f16: return fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(RetVT, Op0, Op1);
7097 case MVT::v2f32: return fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(RetVT, Op0, Op1);
7098 case MVT::v4f32: return fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(RetVT, Op0, Op1);
7099 case MVT::v2f64: return fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(RetVT, Op0, Op1);
7100 default: return Register();
7101 }
7102}
7103
7104// FastEmit functions for AArch64ISD::BIC.
7105
7106Register fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7107 if (RetVT.SimpleTy != MVT::nxv16i8)
7108 return Register();
7109 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7110 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
7111 }
7112 return Register();
7113}
7114
7115Register fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7116 if (RetVT.SimpleTy != MVT::nxv8i16)
7117 return Register();
7118 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7119 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
7120 }
7121 return Register();
7122}
7123
7124Register fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7125 if (RetVT.SimpleTy != MVT::nxv4i32)
7126 return Register();
7127 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7128 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
7129 }
7130 return Register();
7131}
7132
7133Register fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7134 if (RetVT.SimpleTy != MVT::nxv2i64)
7135 return Register();
7136 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7137 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
7138 }
7139 return Register();
7140}
7141
7142Register fastEmit_AArch64ISD_BIC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7143 switch (VT.SimpleTy) {
7144 case MVT::nxv16i8: return fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(RetVT, Op0, Op1);
7145 case MVT::nxv8i16: return fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(RetVT, Op0, Op1);
7146 case MVT::nxv4i32: return fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(RetVT, Op0, Op1);
7147 case MVT::nxv2i64: return fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(RetVT, Op0, Op1);
7148 default: return Register();
7149 }
7150}
7151
7152// FastEmit functions for AArch64ISD::FCMEQ.
7153
7154Register fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7155 if (RetVT.SimpleTy != MVT::i32)
7156 return Register();
7157 if ((Subtarget->isNeonAvailable())) {
7158 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7159 }
7160 return Register();
7161}
7162
7163Register fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7164 if (RetVT.SimpleTy != MVT::i64)
7165 return Register();
7166 if ((Subtarget->isNeonAvailable())) {
7167 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7168 }
7169 return Register();
7170}
7171
7172Register fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7173 if (RetVT.SimpleTy != MVT::v4i16)
7174 return Register();
7175 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7176 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7177 }
7178 return Register();
7179}
7180
7181Register fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7182 if (RetVT.SimpleTy != MVT::v8i16)
7183 return Register();
7184 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7185 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7186 }
7187 return Register();
7188}
7189
7190Register fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7191 if (RetVT.SimpleTy != MVT::v2i32)
7192 return Register();
7193 if ((Subtarget->isNeonAvailable())) {
7194 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7195 }
7196 return Register();
7197}
7198
7199Register fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7200 if (RetVT.SimpleTy != MVT::v4i32)
7201 return Register();
7202 if ((Subtarget->isNeonAvailable())) {
7203 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7204 }
7205 return Register();
7206}
7207
7208Register fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7209 if (RetVT.SimpleTy != MVT::v1i64)
7210 return Register();
7211 if ((Subtarget->isNeonAvailable())) {
7212 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7213 }
7214 return Register();
7215}
7216
7217Register fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7218 if (RetVT.SimpleTy != MVT::v2i64)
7219 return Register();
7220 if ((Subtarget->isNeonAvailable())) {
7221 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7222 }
7223 return Register();
7224}
7225
7226Register fastEmit_AArch64ISD_FCMEQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7227 switch (VT.SimpleTy) {
7228 case MVT::f32: return fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(RetVT, Op0, Op1);
7229 case MVT::f64: return fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(RetVT, Op0, Op1);
7230 case MVT::v4f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(RetVT, Op0, Op1);
7231 case MVT::v8f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(RetVT, Op0, Op1);
7232 case MVT::v2f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(RetVT, Op0, Op1);
7233 case MVT::v4f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(RetVT, Op0, Op1);
7234 case MVT::v1f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(RetVT, Op0, Op1);
7235 case MVT::v2f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(RetVT, Op0, Op1);
7236 default: return Register();
7237 }
7238}
7239
7240// FastEmit functions for AArch64ISD::FCMGE.
7241
7242Register fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7243 if (RetVT.SimpleTy != MVT::i32)
7244 return Register();
7245 if ((Subtarget->isNeonAvailable())) {
7246 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7247 }
7248 return Register();
7249}
7250
7251Register fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7252 if (RetVT.SimpleTy != MVT::i64)
7253 return Register();
7254 if ((Subtarget->isNeonAvailable())) {
7255 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7256 }
7257 return Register();
7258}
7259
7260Register fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7261 if (RetVT.SimpleTy != MVT::v4i16)
7262 return Register();
7263 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7264 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7265 }
7266 return Register();
7267}
7268
7269Register fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7270 if (RetVT.SimpleTy != MVT::v8i16)
7271 return Register();
7272 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7273 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7274 }
7275 return Register();
7276}
7277
7278Register fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7279 if (RetVT.SimpleTy != MVT::v2i32)
7280 return Register();
7281 if ((Subtarget->isNeonAvailable())) {
7282 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7283 }
7284 return Register();
7285}
7286
7287Register fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7288 if (RetVT.SimpleTy != MVT::v4i32)
7289 return Register();
7290 if ((Subtarget->isNeonAvailable())) {
7291 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7292 }
7293 return Register();
7294}
7295
7296Register fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7297 if (RetVT.SimpleTy != MVT::v1i64)
7298 return Register();
7299 if ((Subtarget->isNeonAvailable())) {
7300 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7301 }
7302 return Register();
7303}
7304
7305Register fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7306 if (RetVT.SimpleTy != MVT::v2i64)
7307 return Register();
7308 if ((Subtarget->isNeonAvailable())) {
7309 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7310 }
7311 return Register();
7312}
7313
7314Register fastEmit_AArch64ISD_FCMGE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7315 switch (VT.SimpleTy) {
7316 case MVT::f32: return fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(RetVT, Op0, Op1);
7317 case MVT::f64: return fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(RetVT, Op0, Op1);
7318 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(RetVT, Op0, Op1);
7319 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(RetVT, Op0, Op1);
7320 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(RetVT, Op0, Op1);
7321 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(RetVT, Op0, Op1);
7322 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(RetVT, Op0, Op1);
7323 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(RetVT, Op0, Op1);
7324 default: return Register();
7325 }
7326}
7327
7328// FastEmit functions for AArch64ISD::FCMGT.
7329
7330Register fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7331 if (RetVT.SimpleTy != MVT::i32)
7332 return Register();
7333 if ((Subtarget->isNeonAvailable())) {
7334 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7335 }
7336 return Register();
7337}
7338
7339Register fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7340 if (RetVT.SimpleTy != MVT::i64)
7341 return Register();
7342 if ((Subtarget->isNeonAvailable())) {
7343 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7344 }
7345 return Register();
7346}
7347
7348Register fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7349 if (RetVT.SimpleTy != MVT::v4i16)
7350 return Register();
7351 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7352 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7353 }
7354 return Register();
7355}
7356
7357Register fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7358 if (RetVT.SimpleTy != MVT::v8i16)
7359 return Register();
7360 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7361 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7362 }
7363 return Register();
7364}
7365
7366Register fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7367 if (RetVT.SimpleTy != MVT::v2i32)
7368 return Register();
7369 if ((Subtarget->isNeonAvailable())) {
7370 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7371 }
7372 return Register();
7373}
7374
7375Register fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7376 if (RetVT.SimpleTy != MVT::v4i32)
7377 return Register();
7378 if ((Subtarget->isNeonAvailable())) {
7379 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7380 }
7381 return Register();
7382}
7383
7384Register fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7385 if (RetVT.SimpleTy != MVT::v1i64)
7386 return Register();
7387 if ((Subtarget->isNeonAvailable())) {
7388 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7389 }
7390 return Register();
7391}
7392
7393Register fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7394 if (RetVT.SimpleTy != MVT::v2i64)
7395 return Register();
7396 if ((Subtarget->isNeonAvailable())) {
7397 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7398 }
7399 return Register();
7400}
7401
7402Register fastEmit_AArch64ISD_FCMGT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7403 switch (VT.SimpleTy) {
7404 case MVT::f32: return fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(RetVT, Op0, Op1);
7405 case MVT::f64: return fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(RetVT, Op0, Op1);
7406 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(RetVT, Op0, Op1);
7407 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(RetVT, Op0, Op1);
7408 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(RetVT, Op0, Op1);
7409 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(RetVT, Op0, Op1);
7410 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(RetVT, Op0, Op1);
7411 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(RetVT, Op0, Op1);
7412 default: return Register();
7413 }
7414}
7415
7416// FastEmit functions for AArch64ISD::FCMP.
7417
7418Register fastEmit_AArch64ISD_FCMP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
7419 if (RetVT.SimpleTy != MVT::i32)
7420 return Register();
7421 if ((Subtarget->hasFullFP16())) {
7422 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
7423 }
7424 return Register();
7425}
7426
7427Register fastEmit_AArch64ISD_FCMP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7428 if (RetVT.SimpleTy != MVT::i32)
7429 return Register();
7430 if ((Subtarget->hasFPARMv8())) {
7431 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
7432 }
7433 return Register();
7434}
7435
7436Register fastEmit_AArch64ISD_FCMP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7437 if (RetVT.SimpleTy != MVT::i32)
7438 return Register();
7439 if ((Subtarget->hasFPARMv8())) {
7440 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
7441 }
7442 return Register();
7443}
7444
7445Register fastEmit_AArch64ISD_FCMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7446 switch (VT.SimpleTy) {
7447 case MVT::f16: return fastEmit_AArch64ISD_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
7448 case MVT::f32: return fastEmit_AArch64ISD_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
7449 case MVT::f64: return fastEmit_AArch64ISD_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
7450 default: return Register();
7451 }
7452}
7453
7454// FastEmit functions for AArch64ISD::FRECPS.
7455
7456Register fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7457 if (RetVT.SimpleTy != MVT::f32)
7458 return Register();
7459 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7460}
7461
7462Register fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7463 if (RetVT.SimpleTy != MVT::f64)
7464 return Register();
7465 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7466}
7467
7468Register fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7469 if (RetVT.SimpleTy != MVT::v2f32)
7470 return Register();
7471 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7472}
7473
7474Register fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7475 if (RetVT.SimpleTy != MVT::v4f32)
7476 return Register();
7477 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7478}
7479
7480Register fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7481 if (RetVT.SimpleTy != MVT::v2f64)
7482 return Register();
7483 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7484}
7485
7486Register fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7487 if (RetVT.SimpleTy != MVT::nxv8f16)
7488 return Register();
7489 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7490 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7491 }
7492 return Register();
7493}
7494
7495Register fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7496 if (RetVT.SimpleTy != MVT::nxv4f32)
7497 return Register();
7498 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7499 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7500 }
7501 return Register();
7502}
7503
7504Register fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7505 if (RetVT.SimpleTy != MVT::nxv2f64)
7506 return Register();
7507 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7508 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7509 }
7510 return Register();
7511}
7512
7513Register fastEmit_AArch64ISD_FRECPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7514 switch (VT.SimpleTy) {
7515 case MVT::f32: return fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(RetVT, Op0, Op1);
7516 case MVT::f64: return fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(RetVT, Op0, Op1);
7517 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(RetVT, Op0, Op1);
7518 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(RetVT, Op0, Op1);
7519 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(RetVT, Op0, Op1);
7520 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(RetVT, Op0, Op1);
7521 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(RetVT, Op0, Op1);
7522 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(RetVT, Op0, Op1);
7523 default: return Register();
7524 }
7525}
7526
7527// FastEmit functions for AArch64ISD::FRSQRTS.
7528
7529Register fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7530 if (RetVT.SimpleTy != MVT::f32)
7531 return Register();
7532 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7533}
7534
7535Register fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7536 if (RetVT.SimpleTy != MVT::f64)
7537 return Register();
7538 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7539}
7540
7541Register fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7542 if (RetVT.SimpleTy != MVT::v2f32)
7543 return Register();
7544 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7545}
7546
7547Register fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7548 if (RetVT.SimpleTy != MVT::v4f32)
7549 return Register();
7550 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7551}
7552
7553Register fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7554 if (RetVT.SimpleTy != MVT::v2f64)
7555 return Register();
7556 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7557}
7558
7559Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7560 if (RetVT.SimpleTy != MVT::nxv8f16)
7561 return Register();
7562 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7563 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7564 }
7565 return Register();
7566}
7567
7568Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7569 if (RetVT.SimpleTy != MVT::nxv4f32)
7570 return Register();
7571 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7572 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7573 }
7574 return Register();
7575}
7576
7577Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7578 if (RetVT.SimpleTy != MVT::nxv2f64)
7579 return Register();
7580 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7581 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7582 }
7583 return Register();
7584}
7585
7586Register fastEmit_AArch64ISD_FRSQRTS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7587 switch (VT.SimpleTy) {
7588 case MVT::f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(RetVT, Op0, Op1);
7589 case MVT::f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(RetVT, Op0, Op1);
7590 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(RetVT, Op0, Op1);
7591 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(RetVT, Op0, Op1);
7592 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(RetVT, Op0, Op1);
7593 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(RetVT, Op0, Op1);
7594 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(RetVT, Op0, Op1);
7595 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(RetVT, Op0, Op1);
7596 default: return Register();
7597 }
7598}
7599
7600// FastEmit functions for AArch64ISD::INIT_TPIDR2OBJ.
7601
7602Register fastEmit_AArch64ISD_INIT_TPIDR2OBJ_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
7603 if (RetVT.SimpleTy != MVT::isVoid)
7604 return Register();
7605 return fastEmitInst_rr(MachineInstOpcode: AArch64::InitTPIDR2Obj, RC: &AArch64::GPR64RegClass, Op0, Op1);
7606}
7607
7608Register fastEmit_AArch64ISD_INIT_TPIDR2OBJ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7609 switch (VT.SimpleTy) {
7610 case MVT::i64: return fastEmit_AArch64ISD_INIT_TPIDR2OBJ_MVT_i64_rr(RetVT, Op0, Op1);
7611 default: return Register();
7612 }
7613}
7614
7615// FastEmit functions for AArch64ISD::PMULL.
7616
7617Register fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7618 if (RetVT.SimpleTy != MVT::v8i16)
7619 return Register();
7620 if ((Subtarget->isNeonAvailable())) {
7621 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULLv8i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
7622 }
7623 return Register();
7624}
7625
7626Register fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7627 if (RetVT.SimpleTy != MVT::v16i8)
7628 return Register();
7629 if ((Subtarget->hasAES())) {
7630 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULLv1i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7631 }
7632 return Register();
7633}
7634
7635Register fastEmit_AArch64ISD_PMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7636 switch (VT.SimpleTy) {
7637 case MVT::v8i8: return fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
7638 case MVT::v1i64: return fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(RetVT, Op0, Op1);
7639 default: return Register();
7640 }
7641}
7642
7643// FastEmit functions for AArch64ISD::PTEST.
7644
7645Register fastEmit_AArch64ISD_PTEST_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7646 if (RetVT.SimpleTy != MVT::i32)
7647 return Register();
7648 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7649 return fastEmitInst_rr(MachineInstOpcode: AArch64::PTEST_PP, RC: &AArch64::PPRRegClass, Op0, Op1);
7650 }
7651 return Register();
7652}
7653
7654Register fastEmit_AArch64ISD_PTEST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7655 switch (VT.SimpleTy) {
7656 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7657 default: return Register();
7658 }
7659}
7660
7661// FastEmit functions for AArch64ISD::PTEST_ANY.
7662
7663Register fastEmit_AArch64ISD_PTEST_ANY_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7664 if (RetVT.SimpleTy != MVT::i32)
7665 return Register();
7666 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7667 return fastEmitInst_rr(MachineInstOpcode: AArch64::PTEST_PP_ANY, RC: &AArch64::PPRRegClass, Op0, Op1);
7668 }
7669 return Register();
7670}
7671
7672Register fastEmit_AArch64ISD_PTEST_ANY_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7673 switch (VT.SimpleTy) {
7674 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_ANY_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7675 default: return Register();
7676 }
7677}
7678
7679// FastEmit functions for AArch64ISD::PTEST_FIRST.
7680
7681Register fastEmit_AArch64ISD_PTEST_FIRST_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7682 if (RetVT.SimpleTy != MVT::i32)
7683 return Register();
7684 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7685 return fastEmitInst_rr(MachineInstOpcode: AArch64::PTEST_PP_FIRST, RC: &AArch64::PPRRegClass, Op0, Op1);
7686 }
7687 return Register();
7688}
7689
7690Register fastEmit_AArch64ISD_PTEST_FIRST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7691 switch (VT.SimpleTy) {
7692 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_FIRST_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7693 default: return Register();
7694 }
7695}
7696
7697// FastEmit functions for AArch64ISD::SMULL.
7698
7699Register fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7700 if (RetVT.SimpleTy != MVT::v8i16)
7701 return Register();
7702 if ((Subtarget->isNeonAvailable())) {
7703 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv8i8_v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7704 }
7705 return Register();
7706}
7707
7708Register fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7709 if (RetVT.SimpleTy != MVT::v4i32)
7710 return Register();
7711 if ((Subtarget->isNeonAvailable())) {
7712 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv4i16_v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7713 }
7714 return Register();
7715}
7716
7717Register fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7718 if (RetVT.SimpleTy != MVT::v2i64)
7719 return Register();
7720 if ((Subtarget->isNeonAvailable())) {
7721 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv2i32_v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7722 }
7723 return Register();
7724}
7725
7726Register fastEmit_AArch64ISD_SMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7727 switch (VT.SimpleTy) {
7728 case MVT::v8i8: return fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
7729 case MVT::v4i16: return fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(RetVT, Op0, Op1);
7730 case MVT::v2i32: return fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(RetVT, Op0, Op1);
7731 default: return Register();
7732 }
7733}
7734
7735// FastEmit functions for AArch64ISD::SQADD.
7736
7737Register fastEmit_AArch64ISD_SQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7738 if (RetVT.SimpleTy != MVT::f32)
7739 return Register();
7740 if ((Subtarget->isNeonAvailable())) {
7741 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7742 }
7743 return Register();
7744}
7745
7746Register fastEmit_AArch64ISD_SQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7747 if (RetVT.SimpleTy != MVT::f64)
7748 return Register();
7749 if ((Subtarget->isNeonAvailable())) {
7750 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7751 }
7752 return Register();
7753}
7754
7755Register fastEmit_AArch64ISD_SQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7756 switch (VT.SimpleTy) {
7757 case MVT::f32: return fastEmit_AArch64ISD_SQADD_MVT_f32_rr(RetVT, Op0, Op1);
7758 case MVT::f64: return fastEmit_AArch64ISD_SQADD_MVT_f64_rr(RetVT, Op0, Op1);
7759 default: return Register();
7760 }
7761}
7762
7763// FastEmit functions for AArch64ISD::SQDMULH.
7764
7765Register fastEmit_AArch64ISD_SQDMULH_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7766 if (RetVT.SimpleTy != MVT::f32)
7767 return Register();
7768 if ((Subtarget->isNeonAvailable())) {
7769 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7770 }
7771 return Register();
7772}
7773
7774Register fastEmit_AArch64ISD_SQDMULH_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7775 if (RetVT.SimpleTy != MVT::v4i16)
7776 return Register();
7777 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7778}
7779
7780Register fastEmit_AArch64ISD_SQDMULH_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7781 if (RetVT.SimpleTy != MVT::v8i16)
7782 return Register();
7783 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7784}
7785
7786Register fastEmit_AArch64ISD_SQDMULH_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7787 if (RetVT.SimpleTy != MVT::v2i32)
7788 return Register();
7789 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7790}
7791
7792Register fastEmit_AArch64ISD_SQDMULH_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7793 if (RetVT.SimpleTy != MVT::v4i32)
7794 return Register();
7795 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7796}
7797
7798Register fastEmit_AArch64ISD_SQDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7799 switch (VT.SimpleTy) {
7800 case MVT::f32: return fastEmit_AArch64ISD_SQDMULH_MVT_f32_rr(RetVT, Op0, Op1);
7801 case MVT::v4i16: return fastEmit_AArch64ISD_SQDMULH_MVT_v4i16_rr(RetVT, Op0, Op1);
7802 case MVT::v8i16: return fastEmit_AArch64ISD_SQDMULH_MVT_v8i16_rr(RetVT, Op0, Op1);
7803 case MVT::v2i32: return fastEmit_AArch64ISD_SQDMULH_MVT_v2i32_rr(RetVT, Op0, Op1);
7804 case MVT::v4i32: return fastEmit_AArch64ISD_SQDMULH_MVT_v4i32_rr(RetVT, Op0, Op1);
7805 default: return Register();
7806 }
7807}
7808
7809// FastEmit functions for AArch64ISD::SQDMULL.
7810
7811Register fastEmit_AArch64ISD_SQDMULL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7812 if (RetVT.SimpleTy != MVT::f64)
7813 return Register();
7814 if ((Subtarget->isNeonAvailable())) {
7815 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULLi32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7816 }
7817 return Register();
7818}
7819
7820Register fastEmit_AArch64ISD_SQDMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7821 switch (VT.SimpleTy) {
7822 case MVT::f32: return fastEmit_AArch64ISD_SQDMULL_MVT_f32_rr(RetVT, Op0, Op1);
7823 default: return Register();
7824 }
7825}
7826
7827// FastEmit functions for AArch64ISD::SQRDMULH.
7828
7829Register fastEmit_AArch64ISD_SQRDMULH_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7830 if (RetVT.SimpleTy != MVT::f32)
7831 return Register();
7832 if ((Subtarget->isNeonAvailable())) {
7833 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRDMULHv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7834 }
7835 return Register();
7836}
7837
7838Register fastEmit_AArch64ISD_SQRDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7839 switch (VT.SimpleTy) {
7840 case MVT::f32: return fastEmit_AArch64ISD_SQRDMULH_MVT_f32_rr(RetVT, Op0, Op1);
7841 default: return Register();
7842 }
7843}
7844
7845// FastEmit functions for AArch64ISD::SQRSHL.
7846
7847Register fastEmit_AArch64ISD_SQRSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7848 if (RetVT.SimpleTy != MVT::f32)
7849 return Register();
7850 if ((Subtarget->isNeonAvailable())) {
7851 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7852 }
7853 return Register();
7854}
7855
7856Register fastEmit_AArch64ISD_SQRSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7857 if (RetVT.SimpleTy != MVT::f64)
7858 return Register();
7859 if ((Subtarget->isNeonAvailable())) {
7860 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7861 }
7862 return Register();
7863}
7864
7865Register fastEmit_AArch64ISD_SQRSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7866 switch (VT.SimpleTy) {
7867 case MVT::f32: return fastEmit_AArch64ISD_SQRSHL_MVT_f32_rr(RetVT, Op0, Op1);
7868 case MVT::f64: return fastEmit_AArch64ISD_SQRSHL_MVT_f64_rr(RetVT, Op0, Op1);
7869 default: return Register();
7870 }
7871}
7872
7873// FastEmit functions for AArch64ISD::SQSHL.
7874
7875Register fastEmit_AArch64ISD_SQSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7876 if (RetVT.SimpleTy != MVT::f32)
7877 return Register();
7878 if ((Subtarget->isNeonAvailable())) {
7879 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7880 }
7881 return Register();
7882}
7883
7884Register fastEmit_AArch64ISD_SQSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7885 if (RetVT.SimpleTy != MVT::f64)
7886 return Register();
7887 if ((Subtarget->isNeonAvailable())) {
7888 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7889 }
7890 return Register();
7891}
7892
7893Register fastEmit_AArch64ISD_SQSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7894 switch (VT.SimpleTy) {
7895 case MVT::f32: return fastEmit_AArch64ISD_SQSHL_MVT_f32_rr(RetVT, Op0, Op1);
7896 case MVT::f64: return fastEmit_AArch64ISD_SQSHL_MVT_f64_rr(RetVT, Op0, Op1);
7897 default: return Register();
7898 }
7899}
7900
7901// FastEmit functions for AArch64ISD::SQSUB.
7902
7903Register fastEmit_AArch64ISD_SQSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7904 if (RetVT.SimpleTy != MVT::f32)
7905 return Register();
7906 if ((Subtarget->isNeonAvailable())) {
7907 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7908 }
7909 return Register();
7910}
7911
7912Register fastEmit_AArch64ISD_SQSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7913 if (RetVT.SimpleTy != MVT::f64)
7914 return Register();
7915 if ((Subtarget->isNeonAvailable())) {
7916 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7917 }
7918 return Register();
7919}
7920
7921Register fastEmit_AArch64ISD_SQSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7922 switch (VT.SimpleTy) {
7923 case MVT::f32: return fastEmit_AArch64ISD_SQSUB_MVT_f32_rr(RetVT, Op0, Op1);
7924 case MVT::f64: return fastEmit_AArch64ISD_SQSUB_MVT_f64_rr(RetVT, Op0, Op1);
7925 default: return Register();
7926 }
7927}
7928
7929// FastEmit functions for AArch64ISD::STRICT_FCMP.
7930
7931Register fastEmit_AArch64ISD_STRICT_FCMP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
7932 if (RetVT.SimpleTy != MVT::i32)
7933 return Register();
7934 if ((Subtarget->hasFullFP16())) {
7935 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
7936 }
7937 return Register();
7938}
7939
7940Register fastEmit_AArch64ISD_STRICT_FCMP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7941 if (RetVT.SimpleTy != MVT::i32)
7942 return Register();
7943 if ((Subtarget->hasFPARMv8())) {
7944 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
7945 }
7946 return Register();
7947}
7948
7949Register fastEmit_AArch64ISD_STRICT_FCMP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7950 if (RetVT.SimpleTy != MVT::i32)
7951 return Register();
7952 if ((Subtarget->hasFPARMv8())) {
7953 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
7954 }
7955 return Register();
7956}
7957
7958Register fastEmit_AArch64ISD_STRICT_FCMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7959 switch (VT.SimpleTy) {
7960 case MVT::f16: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
7961 case MVT::f32: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
7962 case MVT::f64: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
7963 default: return Register();
7964 }
7965}
7966
7967// FastEmit functions for AArch64ISD::STRICT_FCMPE.
7968
7969Register fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
7970 if (RetVT.SimpleTy != MVT::i32)
7971 return Register();
7972 if ((Subtarget->hasFullFP16())) {
7973 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPEHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
7974 }
7975 return Register();
7976}
7977
7978Register fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7979 if (RetVT.SimpleTy != MVT::i32)
7980 return Register();
7981 if ((Subtarget->hasFPARMv8())) {
7982 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPESrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
7983 }
7984 return Register();
7985}
7986
7987Register fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7988 if (RetVT.SimpleTy != MVT::i32)
7989 return Register();
7990 if ((Subtarget->hasFPARMv8())) {
7991 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPEDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
7992 }
7993 return Register();
7994}
7995
7996Register fastEmit_AArch64ISD_STRICT_FCMPE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7997 switch (VT.SimpleTy) {
7998 case MVT::f16: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f16_rr(RetVT, Op0, Op1);
7999 case MVT::f32: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f32_rr(RetVT, Op0, Op1);
8000 case MVT::f64: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f64_rr(RetVT, Op0, Op1);
8001 default: return Register();
8002 }
8003}
8004
8005// FastEmit functions for AArch64ISD::SUQADD.
8006
8007Register fastEmit_AArch64ISD_SUQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8008 if (RetVT.SimpleTy != MVT::f32)
8009 return Register();
8010 if ((Subtarget->isNeonAvailable())) {
8011 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8012 }
8013 return Register();
8014}
8015
8016Register fastEmit_AArch64ISD_SUQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8017 if (RetVT.SimpleTy != MVT::f64)
8018 return Register();
8019 if ((Subtarget->isNeonAvailable())) {
8020 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8021 }
8022 return Register();
8023}
8024
8025Register fastEmit_AArch64ISD_SUQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8026 switch (VT.SimpleTy) {
8027 case MVT::f32: return fastEmit_AArch64ISD_SUQADD_MVT_f32_rr(RetVT, Op0, Op1);
8028 case MVT::f64: return fastEmit_AArch64ISD_SUQADD_MVT_f64_rr(RetVT, Op0, Op1);
8029 default: return Register();
8030 }
8031}
8032
8033// FastEmit functions for AArch64ISD::TBL.
8034
8035Register fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8036 if (RetVT.SimpleTy != MVT::nxv16i8)
8037 return Register();
8038 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8039 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8040 }
8041 return Register();
8042}
8043
8044Register fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8045 if (RetVT.SimpleTy != MVT::nxv8i16)
8046 return Register();
8047 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8048 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8049 }
8050 return Register();
8051}
8052
8053Register fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8054 if (RetVT.SimpleTy != MVT::nxv4i32)
8055 return Register();
8056 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8057 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8058 }
8059 return Register();
8060}
8061
8062Register fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8063 if (RetVT.SimpleTy != MVT::nxv2i64)
8064 return Register();
8065 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8066 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8067 }
8068 return Register();
8069}
8070
8071Register fastEmit_AArch64ISD_TBL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8072 switch (VT.SimpleTy) {
8073 case MVT::nxv16i8: return fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8074 case MVT::nxv8i16: return fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8075 case MVT::nxv4i32: return fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8076 case MVT::nxv2i64: return fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8077 default: return Register();
8078 }
8079}
8080
8081// FastEmit functions for AArch64ISD::TRN1.
8082
8083Register fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8084 if (RetVT.SimpleTy != MVT::v8i8)
8085 return Register();
8086 if ((Subtarget->isNeonAvailable())) {
8087 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8088 }
8089 return Register();
8090}
8091
8092Register fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8093 if (RetVT.SimpleTy != MVT::v16i8)
8094 return Register();
8095 if ((Subtarget->isNeonAvailable())) {
8096 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8097 }
8098 return Register();
8099}
8100
8101Register fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8102 if (RetVT.SimpleTy != MVT::v4i16)
8103 return Register();
8104 if ((Subtarget->isNeonAvailable())) {
8105 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8106 }
8107 return Register();
8108}
8109
8110Register fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8111 if (RetVT.SimpleTy != MVT::v8i16)
8112 return Register();
8113 if ((Subtarget->isNeonAvailable())) {
8114 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8115 }
8116 return Register();
8117}
8118
8119Register fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8120 if (RetVT.SimpleTy != MVT::v2i32)
8121 return Register();
8122 if ((Subtarget->isNeonAvailable())) {
8123 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8124 }
8125 return Register();
8126}
8127
8128Register fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8129 if (RetVT.SimpleTy != MVT::v4i32)
8130 return Register();
8131 if ((Subtarget->isNeonAvailable())) {
8132 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8133 }
8134 return Register();
8135}
8136
8137Register fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8138 if (RetVT.SimpleTy != MVT::v2i64)
8139 return Register();
8140 if ((Subtarget->isNeonAvailable())) {
8141 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8142 }
8143 return Register();
8144}
8145
8146Register fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8147 if (RetVT.SimpleTy != MVT::v4f16)
8148 return Register();
8149 if ((Subtarget->isNeonAvailable())) {
8150 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8151 }
8152 return Register();
8153}
8154
8155Register fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8156 if (RetVT.SimpleTy != MVT::v8f16)
8157 return Register();
8158 if ((Subtarget->isNeonAvailable())) {
8159 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8160 }
8161 return Register();
8162}
8163
8164Register fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8165 if (RetVT.SimpleTy != MVT::v4bf16)
8166 return Register();
8167 if ((Subtarget->isNeonAvailable())) {
8168 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8169 }
8170 return Register();
8171}
8172
8173Register fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8174 if (RetVT.SimpleTy != MVT::v8bf16)
8175 return Register();
8176 if ((Subtarget->isNeonAvailable())) {
8177 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8178 }
8179 return Register();
8180}
8181
8182Register fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8183 if (RetVT.SimpleTy != MVT::v2f32)
8184 return Register();
8185 if ((Subtarget->isNeonAvailable())) {
8186 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8187 }
8188 return Register();
8189}
8190
8191Register fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8192 if (RetVT.SimpleTy != MVT::v4f32)
8193 return Register();
8194 if ((Subtarget->isNeonAvailable())) {
8195 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8196 }
8197 return Register();
8198}
8199
8200Register fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8201 if (RetVT.SimpleTy != MVT::v2f64)
8202 return Register();
8203 if ((Subtarget->isNeonAvailable())) {
8204 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8205 }
8206 return Register();
8207}
8208
8209Register fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8210 if (RetVT.SimpleTy != MVT::nxv2i1)
8211 return Register();
8212 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8213 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8214 }
8215 return Register();
8216}
8217
8218Register fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8219 if (RetVT.SimpleTy != MVT::nxv4i1)
8220 return Register();
8221 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8222 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8223 }
8224 return Register();
8225}
8226
8227Register fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8228 if (RetVT.SimpleTy != MVT::nxv8i1)
8229 return Register();
8230 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8231 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8232 }
8233 return Register();
8234}
8235
8236Register fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8237 if (RetVT.SimpleTy != MVT::nxv16i1)
8238 return Register();
8239 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8240 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8241 }
8242 return Register();
8243}
8244
8245Register fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8246 if (RetVT.SimpleTy != MVT::nxv16i8)
8247 return Register();
8248 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8249 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8250 }
8251 return Register();
8252}
8253
8254Register fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8255 if (RetVT.SimpleTy != MVT::nxv8i16)
8256 return Register();
8257 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8258 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8259 }
8260 return Register();
8261}
8262
8263Register fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8264 if (RetVT.SimpleTy != MVT::nxv4i32)
8265 return Register();
8266 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8267 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8268 }
8269 return Register();
8270}
8271
8272Register fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8273 if (RetVT.SimpleTy != MVT::nxv2i64)
8274 return Register();
8275 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8276 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8277 }
8278 return Register();
8279}
8280
8281Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8282 if (RetVT.SimpleTy != MVT::nxv2f16)
8283 return Register();
8284 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8285 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8286 }
8287 return Register();
8288}
8289
8290Register fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8291 if (RetVT.SimpleTy != MVT::nxv4f16)
8292 return Register();
8293 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8294 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8295 }
8296 return Register();
8297}
8298
8299Register fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8300 if (RetVT.SimpleTy != MVT::nxv8f16)
8301 return Register();
8302 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8303 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8304 }
8305 return Register();
8306}
8307
8308Register fastEmit_AArch64ISD_TRN1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8309 if (RetVT.SimpleTy != MVT::nxv2bf16)
8310 return Register();
8311 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8312 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8313 }
8314 return Register();
8315}
8316
8317Register fastEmit_AArch64ISD_TRN1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8318 if (RetVT.SimpleTy != MVT::nxv4bf16)
8319 return Register();
8320 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8321 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8322 }
8323 return Register();
8324}
8325
8326Register fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8327 if (RetVT.SimpleTy != MVT::nxv8bf16)
8328 return Register();
8329 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8330 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8331 }
8332 return Register();
8333}
8334
8335Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8336 if (RetVT.SimpleTy != MVT::nxv2f32)
8337 return Register();
8338 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8339 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8340 }
8341 return Register();
8342}
8343
8344Register fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8345 if (RetVT.SimpleTy != MVT::nxv4f32)
8346 return Register();
8347 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8348 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8349 }
8350 return Register();
8351}
8352
8353Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8354 if (RetVT.SimpleTy != MVT::nxv2f64)
8355 return Register();
8356 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8357 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8358 }
8359 return Register();
8360}
8361
8362Register fastEmit_AArch64ISD_TRN1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8363 switch (VT.SimpleTy) {
8364 case MVT::v8i8: return fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(RetVT, Op0, Op1);
8365 case MVT::v16i8: return fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(RetVT, Op0, Op1);
8366 case MVT::v4i16: return fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(RetVT, Op0, Op1);
8367 case MVT::v8i16: return fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(RetVT, Op0, Op1);
8368 case MVT::v2i32: return fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(RetVT, Op0, Op1);
8369 case MVT::v4i32: return fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(RetVT, Op0, Op1);
8370 case MVT::v2i64: return fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(RetVT, Op0, Op1);
8371 case MVT::v4f16: return fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(RetVT, Op0, Op1);
8372 case MVT::v8f16: return fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(RetVT, Op0, Op1);
8373 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(RetVT, Op0, Op1);
8374 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(RetVT, Op0, Op1);
8375 case MVT::v2f32: return fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(RetVT, Op0, Op1);
8376 case MVT::v4f32: return fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(RetVT, Op0, Op1);
8377 case MVT::v2f64: return fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(RetVT, Op0, Op1);
8378 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8379 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8380 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8381 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8382 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8383 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8384 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8385 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8386 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8387 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8388 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8389 case MVT::nxv2bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8390 case MVT::nxv4bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8391 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8392 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8393 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8394 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8395 default: return Register();
8396 }
8397}
8398
8399// FastEmit functions for AArch64ISD::TRN2.
8400
8401Register fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8402 if (RetVT.SimpleTy != MVT::v8i8)
8403 return Register();
8404 if ((Subtarget->isNeonAvailable())) {
8405 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8406 }
8407 return Register();
8408}
8409
8410Register fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8411 if (RetVT.SimpleTy != MVT::v16i8)
8412 return Register();
8413 if ((Subtarget->isNeonAvailable())) {
8414 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8415 }
8416 return Register();
8417}
8418
8419Register fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8420 if (RetVT.SimpleTy != MVT::v4i16)
8421 return Register();
8422 if ((Subtarget->isNeonAvailable())) {
8423 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8424 }
8425 return Register();
8426}
8427
8428Register fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8429 if (RetVT.SimpleTy != MVT::v8i16)
8430 return Register();
8431 if ((Subtarget->isNeonAvailable())) {
8432 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8433 }
8434 return Register();
8435}
8436
8437Register fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8438 if (RetVT.SimpleTy != MVT::v2i32)
8439 return Register();
8440 if ((Subtarget->isNeonAvailable())) {
8441 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8442 }
8443 return Register();
8444}
8445
8446Register fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8447 if (RetVT.SimpleTy != MVT::v4i32)
8448 return Register();
8449 if ((Subtarget->isNeonAvailable())) {
8450 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8451 }
8452 return Register();
8453}
8454
8455Register fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8456 if (RetVT.SimpleTy != MVT::v2i64)
8457 return Register();
8458 if ((Subtarget->isNeonAvailable())) {
8459 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8460 }
8461 return Register();
8462}
8463
8464Register fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8465 if (RetVT.SimpleTy != MVT::v4f16)
8466 return Register();
8467 if ((Subtarget->isNeonAvailable())) {
8468 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8469 }
8470 return Register();
8471}
8472
8473Register fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8474 if (RetVT.SimpleTy != MVT::v8f16)
8475 return Register();
8476 if ((Subtarget->isNeonAvailable())) {
8477 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8478 }
8479 return Register();
8480}
8481
8482Register fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8483 if (RetVT.SimpleTy != MVT::v4bf16)
8484 return Register();
8485 if ((Subtarget->isNeonAvailable())) {
8486 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8487 }
8488 return Register();
8489}
8490
8491Register fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8492 if (RetVT.SimpleTy != MVT::v8bf16)
8493 return Register();
8494 if ((Subtarget->isNeonAvailable())) {
8495 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8496 }
8497 return Register();
8498}
8499
8500Register fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8501 if (RetVT.SimpleTy != MVT::v2f32)
8502 return Register();
8503 if ((Subtarget->isNeonAvailable())) {
8504 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8505 }
8506 return Register();
8507}
8508
8509Register fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8510 if (RetVT.SimpleTy != MVT::v4f32)
8511 return Register();
8512 if ((Subtarget->isNeonAvailable())) {
8513 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8514 }
8515 return Register();
8516}
8517
8518Register fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8519 if (RetVT.SimpleTy != MVT::v2f64)
8520 return Register();
8521 if ((Subtarget->isNeonAvailable())) {
8522 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8523 }
8524 return Register();
8525}
8526
8527Register fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8528 if (RetVT.SimpleTy != MVT::nxv2i1)
8529 return Register();
8530 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8531 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8532 }
8533 return Register();
8534}
8535
8536Register fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8537 if (RetVT.SimpleTy != MVT::nxv4i1)
8538 return Register();
8539 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8540 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8541 }
8542 return Register();
8543}
8544
8545Register fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8546 if (RetVT.SimpleTy != MVT::nxv8i1)
8547 return Register();
8548 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8549 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8550 }
8551 return Register();
8552}
8553
8554Register fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8555 if (RetVT.SimpleTy != MVT::nxv16i1)
8556 return Register();
8557 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8558 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8559 }
8560 return Register();
8561}
8562
8563Register fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8564 if (RetVT.SimpleTy != MVT::nxv16i8)
8565 return Register();
8566 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8567 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8568 }
8569 return Register();
8570}
8571
8572Register fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8573 if (RetVT.SimpleTy != MVT::nxv8i16)
8574 return Register();
8575 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8576 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8577 }
8578 return Register();
8579}
8580
8581Register fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8582 if (RetVT.SimpleTy != MVT::nxv4i32)
8583 return Register();
8584 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8585 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8586 }
8587 return Register();
8588}
8589
8590Register fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8591 if (RetVT.SimpleTy != MVT::nxv2i64)
8592 return Register();
8593 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8594 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8595 }
8596 return Register();
8597}
8598
8599Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8600 if (RetVT.SimpleTy != MVT::nxv2f16)
8601 return Register();
8602 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8603 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8604 }
8605 return Register();
8606}
8607
8608Register fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8609 if (RetVT.SimpleTy != MVT::nxv4f16)
8610 return Register();
8611 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8612 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8613 }
8614 return Register();
8615}
8616
8617Register fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8618 if (RetVT.SimpleTy != MVT::nxv8f16)
8619 return Register();
8620 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8621 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8622 }
8623 return Register();
8624}
8625
8626Register fastEmit_AArch64ISD_TRN2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8627 if (RetVT.SimpleTy != MVT::nxv2bf16)
8628 return Register();
8629 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8630 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8631 }
8632 return Register();
8633}
8634
8635Register fastEmit_AArch64ISD_TRN2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8636 if (RetVT.SimpleTy != MVT::nxv4bf16)
8637 return Register();
8638 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8639 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8640 }
8641 return Register();
8642}
8643
8644Register fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8645 if (RetVT.SimpleTy != MVT::nxv8bf16)
8646 return Register();
8647 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8648 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8649 }
8650 return Register();
8651}
8652
8653Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8654 if (RetVT.SimpleTy != MVT::nxv2f32)
8655 return Register();
8656 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8657 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8658 }
8659 return Register();
8660}
8661
8662Register fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8663 if (RetVT.SimpleTy != MVT::nxv4f32)
8664 return Register();
8665 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8666 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8667 }
8668 return Register();
8669}
8670
8671Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8672 if (RetVT.SimpleTy != MVT::nxv2f64)
8673 return Register();
8674 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8675 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8676 }
8677 return Register();
8678}
8679
8680Register fastEmit_AArch64ISD_TRN2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8681 switch (VT.SimpleTy) {
8682 case MVT::v8i8: return fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(RetVT, Op0, Op1);
8683 case MVT::v16i8: return fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(RetVT, Op0, Op1);
8684 case MVT::v4i16: return fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(RetVT, Op0, Op1);
8685 case MVT::v8i16: return fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(RetVT, Op0, Op1);
8686 case MVT::v2i32: return fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(RetVT, Op0, Op1);
8687 case MVT::v4i32: return fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(RetVT, Op0, Op1);
8688 case MVT::v2i64: return fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(RetVT, Op0, Op1);
8689 case MVT::v4f16: return fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(RetVT, Op0, Op1);
8690 case MVT::v8f16: return fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(RetVT, Op0, Op1);
8691 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(RetVT, Op0, Op1);
8692 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(RetVT, Op0, Op1);
8693 case MVT::v2f32: return fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(RetVT, Op0, Op1);
8694 case MVT::v4f32: return fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(RetVT, Op0, Op1);
8695 case MVT::v2f64: return fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(RetVT, Op0, Op1);
8696 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8697 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8698 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8699 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8700 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8701 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8702 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8703 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8704 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8705 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8706 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8707 case MVT::nxv2bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8708 case MVT::nxv4bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8709 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8710 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8711 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8712 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8713 default: return Register();
8714 }
8715}
8716
8717// FastEmit functions for AArch64ISD::UMULL.
8718
8719Register fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8720 if (RetVT.SimpleTy != MVT::v8i16)
8721 return Register();
8722 if ((Subtarget->isNeonAvailable())) {
8723 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv8i8_v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8724 }
8725 return Register();
8726}
8727
8728Register fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8729 if (RetVT.SimpleTy != MVT::v4i32)
8730 return Register();
8731 if ((Subtarget->isNeonAvailable())) {
8732 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv4i16_v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8733 }
8734 return Register();
8735}
8736
8737Register fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8738 if (RetVT.SimpleTy != MVT::v2i64)
8739 return Register();
8740 if ((Subtarget->isNeonAvailable())) {
8741 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv2i32_v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8742 }
8743 return Register();
8744}
8745
8746Register fastEmit_AArch64ISD_UMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8747 switch (VT.SimpleTy) {
8748 case MVT::v8i8: return fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
8749 case MVT::v4i16: return fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(RetVT, Op0, Op1);
8750 case MVT::v2i32: return fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(RetVT, Op0, Op1);
8751 default: return Register();
8752 }
8753}
8754
8755// FastEmit functions for AArch64ISD::UQADD.
8756
8757Register fastEmit_AArch64ISD_UQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8758 if (RetVT.SimpleTy != MVT::f32)
8759 return Register();
8760 if ((Subtarget->isNeonAvailable())) {
8761 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8762 }
8763 return Register();
8764}
8765
8766Register fastEmit_AArch64ISD_UQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8767 if (RetVT.SimpleTy != MVT::f64)
8768 return Register();
8769 if ((Subtarget->isNeonAvailable())) {
8770 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8771 }
8772 return Register();
8773}
8774
8775Register fastEmit_AArch64ISD_UQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8776 switch (VT.SimpleTy) {
8777 case MVT::f32: return fastEmit_AArch64ISD_UQADD_MVT_f32_rr(RetVT, Op0, Op1);
8778 case MVT::f64: return fastEmit_AArch64ISD_UQADD_MVT_f64_rr(RetVT, Op0, Op1);
8779 default: return Register();
8780 }
8781}
8782
8783// FastEmit functions for AArch64ISD::UQRSHL.
8784
8785Register fastEmit_AArch64ISD_UQRSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8786 if (RetVT.SimpleTy != MVT::f32)
8787 return Register();
8788 if ((Subtarget->isNeonAvailable())) {
8789 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQRSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8790 }
8791 return Register();
8792}
8793
8794Register fastEmit_AArch64ISD_UQRSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8795 if (RetVT.SimpleTy != MVT::f64)
8796 return Register();
8797 if ((Subtarget->isNeonAvailable())) {
8798 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQRSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8799 }
8800 return Register();
8801}
8802
8803Register fastEmit_AArch64ISD_UQRSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8804 switch (VT.SimpleTy) {
8805 case MVT::f32: return fastEmit_AArch64ISD_UQRSHL_MVT_f32_rr(RetVT, Op0, Op1);
8806 case MVT::f64: return fastEmit_AArch64ISD_UQRSHL_MVT_f64_rr(RetVT, Op0, Op1);
8807 default: return Register();
8808 }
8809}
8810
8811// FastEmit functions for AArch64ISD::UQSHL.
8812
8813Register fastEmit_AArch64ISD_UQSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8814 if (RetVT.SimpleTy != MVT::f32)
8815 return Register();
8816 if ((Subtarget->isNeonAvailable())) {
8817 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8818 }
8819 return Register();
8820}
8821
8822Register fastEmit_AArch64ISD_UQSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8823 if (RetVT.SimpleTy != MVT::f64)
8824 return Register();
8825 if ((Subtarget->isNeonAvailable())) {
8826 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8827 }
8828 return Register();
8829}
8830
8831Register fastEmit_AArch64ISD_UQSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8832 switch (VT.SimpleTy) {
8833 case MVT::f32: return fastEmit_AArch64ISD_UQSHL_MVT_f32_rr(RetVT, Op0, Op1);
8834 case MVT::f64: return fastEmit_AArch64ISD_UQSHL_MVT_f64_rr(RetVT, Op0, Op1);
8835 default: return Register();
8836 }
8837}
8838
8839// FastEmit functions for AArch64ISD::UQSUB.
8840
8841Register fastEmit_AArch64ISD_UQSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8842 if (RetVT.SimpleTy != MVT::f32)
8843 return Register();
8844 if ((Subtarget->isNeonAvailable())) {
8845 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8846 }
8847 return Register();
8848}
8849
8850Register fastEmit_AArch64ISD_UQSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8851 if (RetVT.SimpleTy != MVT::f64)
8852 return Register();
8853 if ((Subtarget->isNeonAvailable())) {
8854 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8855 }
8856 return Register();
8857}
8858
8859Register fastEmit_AArch64ISD_UQSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8860 switch (VT.SimpleTy) {
8861 case MVT::f32: return fastEmit_AArch64ISD_UQSUB_MVT_f32_rr(RetVT, Op0, Op1);
8862 case MVT::f64: return fastEmit_AArch64ISD_UQSUB_MVT_f64_rr(RetVT, Op0, Op1);
8863 default: return Register();
8864 }
8865}
8866
8867// FastEmit functions for AArch64ISD::USQADD.
8868
8869Register fastEmit_AArch64ISD_USQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8870 if (RetVT.SimpleTy != MVT::f32)
8871 return Register();
8872 if ((Subtarget->isNeonAvailable())) {
8873 return fastEmitInst_rr(MachineInstOpcode: AArch64::USQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8874 }
8875 return Register();
8876}
8877
8878Register fastEmit_AArch64ISD_USQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8879 if (RetVT.SimpleTy != MVT::f64)
8880 return Register();
8881 if ((Subtarget->isNeonAvailable())) {
8882 return fastEmitInst_rr(MachineInstOpcode: AArch64::USQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8883 }
8884 return Register();
8885}
8886
8887Register fastEmit_AArch64ISD_USQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8888 switch (VT.SimpleTy) {
8889 case MVT::f32: return fastEmit_AArch64ISD_USQADD_MVT_f32_rr(RetVT, Op0, Op1);
8890 case MVT::f64: return fastEmit_AArch64ISD_USQADD_MVT_f64_rr(RetVT, Op0, Op1);
8891 default: return Register();
8892 }
8893}
8894
8895// FastEmit functions for AArch64ISD::UZP1.
8896
8897Register fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8898 if (RetVT.SimpleTy != MVT::v8i8)
8899 return Register();
8900 if ((Subtarget->isNeonAvailable())) {
8901 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8902 }
8903 return Register();
8904}
8905
8906Register fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8907 if (RetVT.SimpleTy != MVT::v16i8)
8908 return Register();
8909 if ((Subtarget->isNeonAvailable())) {
8910 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8911 }
8912 return Register();
8913}
8914
8915Register fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8916 if (RetVT.SimpleTy != MVT::v4i16)
8917 return Register();
8918 if ((Subtarget->isNeonAvailable())) {
8919 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8920 }
8921 return Register();
8922}
8923
8924Register fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8925 if (RetVT.SimpleTy != MVT::v8i16)
8926 return Register();
8927 if ((Subtarget->isNeonAvailable())) {
8928 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8929 }
8930 return Register();
8931}
8932
8933Register fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8934 if (RetVT.SimpleTy != MVT::v2i32)
8935 return Register();
8936 if ((Subtarget->isNeonAvailable())) {
8937 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8938 }
8939 return Register();
8940}
8941
8942Register fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8943 if (RetVT.SimpleTy != MVT::v4i32)
8944 return Register();
8945 if ((Subtarget->isNeonAvailable())) {
8946 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8947 }
8948 return Register();
8949}
8950
8951Register fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8952 if (RetVT.SimpleTy != MVT::v2i64)
8953 return Register();
8954 if ((Subtarget->isNeonAvailable())) {
8955 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8956 }
8957 return Register();
8958}
8959
8960Register fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8961 if (RetVT.SimpleTy != MVT::v4f16)
8962 return Register();
8963 if ((Subtarget->isNeonAvailable())) {
8964 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8965 }
8966 return Register();
8967}
8968
8969Register fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8970 if (RetVT.SimpleTy != MVT::v8f16)
8971 return Register();
8972 if ((Subtarget->isNeonAvailable())) {
8973 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8974 }
8975 return Register();
8976}
8977
8978Register fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8979 if (RetVT.SimpleTy != MVT::v4bf16)
8980 return Register();
8981 if ((Subtarget->isNeonAvailable())) {
8982 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8983 }
8984 return Register();
8985}
8986
8987Register fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8988 if (RetVT.SimpleTy != MVT::v8bf16)
8989 return Register();
8990 if ((Subtarget->isNeonAvailable())) {
8991 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8992 }
8993 return Register();
8994}
8995
8996Register fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8997 if (RetVT.SimpleTy != MVT::v2f32)
8998 return Register();
8999 if ((Subtarget->isNeonAvailable())) {
9000 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9001 }
9002 return Register();
9003}
9004
9005Register fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9006 if (RetVT.SimpleTy != MVT::v4f32)
9007 return Register();
9008 if ((Subtarget->isNeonAvailable())) {
9009 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9010 }
9011 return Register();
9012}
9013
9014Register fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9015 if (RetVT.SimpleTy != MVT::v2f64)
9016 return Register();
9017 if ((Subtarget->isNeonAvailable())) {
9018 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9019 }
9020 return Register();
9021}
9022
9023Register fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9024 if (RetVT.SimpleTy != MVT::nxv2i1)
9025 return Register();
9026 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9027 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9028 }
9029 return Register();
9030}
9031
9032Register fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9033 if (RetVT.SimpleTy != MVT::nxv4i1)
9034 return Register();
9035 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9036 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9037 }
9038 return Register();
9039}
9040
9041Register fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9042 if (RetVT.SimpleTy != MVT::nxv8i1)
9043 return Register();
9044 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9045 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9046 }
9047 return Register();
9048}
9049
9050Register fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9051 if (RetVT.SimpleTy != MVT::nxv16i1)
9052 return Register();
9053 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9054 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9055 }
9056 return Register();
9057}
9058
9059Register fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9060 if (RetVT.SimpleTy != MVT::nxv16i8)
9061 return Register();
9062 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9063 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9064 }
9065 return Register();
9066}
9067
9068Register fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9069 if (RetVT.SimpleTy != MVT::nxv8i16)
9070 return Register();
9071 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9072 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9073 }
9074 return Register();
9075}
9076
9077Register fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9078 if (RetVT.SimpleTy != MVT::nxv4i32)
9079 return Register();
9080 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9081 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9082 }
9083 return Register();
9084}
9085
9086Register fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9087 if (RetVT.SimpleTy != MVT::nxv2i64)
9088 return Register();
9089 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9090 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9091 }
9092 return Register();
9093}
9094
9095Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9096 if (RetVT.SimpleTy != MVT::nxv2f16)
9097 return Register();
9098 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9099 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9100 }
9101 return Register();
9102}
9103
9104Register fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9105 if (RetVT.SimpleTy != MVT::nxv4f16)
9106 return Register();
9107 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9108 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9109 }
9110 return Register();
9111}
9112
9113Register fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9114 if (RetVT.SimpleTy != MVT::nxv8f16)
9115 return Register();
9116 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9117 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9118 }
9119 return Register();
9120}
9121
9122Register fastEmit_AArch64ISD_UZP1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9123 if (RetVT.SimpleTy != MVT::nxv2bf16)
9124 return Register();
9125 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9126 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9127 }
9128 return Register();
9129}
9130
9131Register fastEmit_AArch64ISD_UZP1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9132 if (RetVT.SimpleTy != MVT::nxv4bf16)
9133 return Register();
9134 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9135 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9136 }
9137 return Register();
9138}
9139
9140Register fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9141 if (RetVT.SimpleTy != MVT::nxv8bf16)
9142 return Register();
9143 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9144 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9145 }
9146 return Register();
9147}
9148
9149Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9150 if (RetVT.SimpleTy != MVT::nxv2f32)
9151 return Register();
9152 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9153 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9154 }
9155 return Register();
9156}
9157
9158Register fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9159 if (RetVT.SimpleTy != MVT::nxv4f32)
9160 return Register();
9161 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9162 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9163 }
9164 return Register();
9165}
9166
9167Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9168 if (RetVT.SimpleTy != MVT::nxv2f64)
9169 return Register();
9170 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9171 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9172 }
9173 return Register();
9174}
9175
9176Register fastEmit_AArch64ISD_UZP1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9177 switch (VT.SimpleTy) {
9178 case MVT::v8i8: return fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(RetVT, Op0, Op1);
9179 case MVT::v16i8: return fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(RetVT, Op0, Op1);
9180 case MVT::v4i16: return fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(RetVT, Op0, Op1);
9181 case MVT::v8i16: return fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(RetVT, Op0, Op1);
9182 case MVT::v2i32: return fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(RetVT, Op0, Op1);
9183 case MVT::v4i32: return fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(RetVT, Op0, Op1);
9184 case MVT::v2i64: return fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(RetVT, Op0, Op1);
9185 case MVT::v4f16: return fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(RetVT, Op0, Op1);
9186 case MVT::v8f16: return fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(RetVT, Op0, Op1);
9187 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(RetVT, Op0, Op1);
9188 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(RetVT, Op0, Op1);
9189 case MVT::v2f32: return fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(RetVT, Op0, Op1);
9190 case MVT::v4f32: return fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(RetVT, Op0, Op1);
9191 case MVT::v2f64: return fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(RetVT, Op0, Op1);
9192 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9193 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9194 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9195 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9196 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9197 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9198 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9199 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9200 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9201 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9202 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9203 case MVT::nxv2bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9204 case MVT::nxv4bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9205 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9206 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9207 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9208 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9209 default: return Register();
9210 }
9211}
9212
9213// FastEmit functions for AArch64ISD::UZP2.
9214
9215Register fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9216 if (RetVT.SimpleTy != MVT::v8i8)
9217 return Register();
9218 if ((Subtarget->isNeonAvailable())) {
9219 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9220 }
9221 return Register();
9222}
9223
9224Register fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9225 if (RetVT.SimpleTy != MVT::v16i8)
9226 return Register();
9227 if ((Subtarget->isNeonAvailable())) {
9228 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9229 }
9230 return Register();
9231}
9232
9233Register fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9234 if (RetVT.SimpleTy != MVT::v4i16)
9235 return Register();
9236 if ((Subtarget->isNeonAvailable())) {
9237 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9238 }
9239 return Register();
9240}
9241
9242Register fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9243 if (RetVT.SimpleTy != MVT::v8i16)
9244 return Register();
9245 if ((Subtarget->isNeonAvailable())) {
9246 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9247 }
9248 return Register();
9249}
9250
9251Register fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9252 if (RetVT.SimpleTy != MVT::v2i32)
9253 return Register();
9254 if ((Subtarget->isNeonAvailable())) {
9255 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9256 }
9257 return Register();
9258}
9259
9260Register fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9261 if (RetVT.SimpleTy != MVT::v4i32)
9262 return Register();
9263 if ((Subtarget->isNeonAvailable())) {
9264 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9265 }
9266 return Register();
9267}
9268
9269Register fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9270 if (RetVT.SimpleTy != MVT::v2i64)
9271 return Register();
9272 if ((Subtarget->isNeonAvailable())) {
9273 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9274 }
9275 return Register();
9276}
9277
9278Register fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9279 if (RetVT.SimpleTy != MVT::v4f16)
9280 return Register();
9281 if ((Subtarget->isNeonAvailable())) {
9282 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9283 }
9284 return Register();
9285}
9286
9287Register fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9288 if (RetVT.SimpleTy != MVT::v8f16)
9289 return Register();
9290 if ((Subtarget->isNeonAvailable())) {
9291 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9292 }
9293 return Register();
9294}
9295
9296Register fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9297 if (RetVT.SimpleTy != MVT::v4bf16)
9298 return Register();
9299 if ((Subtarget->isNeonAvailable())) {
9300 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9301 }
9302 return Register();
9303}
9304
9305Register fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9306 if (RetVT.SimpleTy != MVT::v8bf16)
9307 return Register();
9308 if ((Subtarget->isNeonAvailable())) {
9309 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9310 }
9311 return Register();
9312}
9313
9314Register fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9315 if (RetVT.SimpleTy != MVT::v2f32)
9316 return Register();
9317 if ((Subtarget->isNeonAvailable())) {
9318 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9319 }
9320 return Register();
9321}
9322
9323Register fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9324 if (RetVT.SimpleTy != MVT::v4f32)
9325 return Register();
9326 if ((Subtarget->isNeonAvailable())) {
9327 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9328 }
9329 return Register();
9330}
9331
9332Register fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9333 if (RetVT.SimpleTy != MVT::v2f64)
9334 return Register();
9335 if ((Subtarget->isNeonAvailable())) {
9336 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9337 }
9338 return Register();
9339}
9340
9341Register fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9342 if (RetVT.SimpleTy != MVT::nxv2i1)
9343 return Register();
9344 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9345 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9346 }
9347 return Register();
9348}
9349
9350Register fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9351 if (RetVT.SimpleTy != MVT::nxv4i1)
9352 return Register();
9353 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9354 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9355 }
9356 return Register();
9357}
9358
9359Register fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9360 if (RetVT.SimpleTy != MVT::nxv8i1)
9361 return Register();
9362 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9363 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9364 }
9365 return Register();
9366}
9367
9368Register fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9369 if (RetVT.SimpleTy != MVT::nxv16i1)
9370 return Register();
9371 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9372 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9373 }
9374 return Register();
9375}
9376
9377Register fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9378 if (RetVT.SimpleTy != MVT::nxv16i8)
9379 return Register();
9380 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9381 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9382 }
9383 return Register();
9384}
9385
9386Register fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9387 if (RetVT.SimpleTy != MVT::nxv8i16)
9388 return Register();
9389 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9390 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9391 }
9392 return Register();
9393}
9394
9395Register fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9396 if (RetVT.SimpleTy != MVT::nxv4i32)
9397 return Register();
9398 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9399 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9400 }
9401 return Register();
9402}
9403
9404Register fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9405 if (RetVT.SimpleTy != MVT::nxv2i64)
9406 return Register();
9407 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9408 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9409 }
9410 return Register();
9411}
9412
9413Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9414 if (RetVT.SimpleTy != MVT::nxv2f16)
9415 return Register();
9416 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9417 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9418 }
9419 return Register();
9420}
9421
9422Register fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9423 if (RetVT.SimpleTy != MVT::nxv4f16)
9424 return Register();
9425 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9426 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9427 }
9428 return Register();
9429}
9430
9431Register fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9432 if (RetVT.SimpleTy != MVT::nxv8f16)
9433 return Register();
9434 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9435 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9436 }
9437 return Register();
9438}
9439
9440Register fastEmit_AArch64ISD_UZP2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9441 if (RetVT.SimpleTy != MVT::nxv2bf16)
9442 return Register();
9443 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9444 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9445 }
9446 return Register();
9447}
9448
9449Register fastEmit_AArch64ISD_UZP2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9450 if (RetVT.SimpleTy != MVT::nxv4bf16)
9451 return Register();
9452 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9453 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9454 }
9455 return Register();
9456}
9457
9458Register fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9459 if (RetVT.SimpleTy != MVT::nxv8bf16)
9460 return Register();
9461 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9462 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9463 }
9464 return Register();
9465}
9466
9467Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9468 if (RetVT.SimpleTy != MVT::nxv2f32)
9469 return Register();
9470 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9471 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9472 }
9473 return Register();
9474}
9475
9476Register fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9477 if (RetVT.SimpleTy != MVT::nxv4f32)
9478 return Register();
9479 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9480 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9481 }
9482 return Register();
9483}
9484
9485Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9486 if (RetVT.SimpleTy != MVT::nxv2f64)
9487 return Register();
9488 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9489 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9490 }
9491 return Register();
9492}
9493
9494Register fastEmit_AArch64ISD_UZP2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9495 switch (VT.SimpleTy) {
9496 case MVT::v8i8: return fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(RetVT, Op0, Op1);
9497 case MVT::v16i8: return fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(RetVT, Op0, Op1);
9498 case MVT::v4i16: return fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(RetVT, Op0, Op1);
9499 case MVT::v8i16: return fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(RetVT, Op0, Op1);
9500 case MVT::v2i32: return fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(RetVT, Op0, Op1);
9501 case MVT::v4i32: return fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(RetVT, Op0, Op1);
9502 case MVT::v2i64: return fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(RetVT, Op0, Op1);
9503 case MVT::v4f16: return fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(RetVT, Op0, Op1);
9504 case MVT::v8f16: return fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(RetVT, Op0, Op1);
9505 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(RetVT, Op0, Op1);
9506 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(RetVT, Op0, Op1);
9507 case MVT::v2f32: return fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(RetVT, Op0, Op1);
9508 case MVT::v4f32: return fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(RetVT, Op0, Op1);
9509 case MVT::v2f64: return fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(RetVT, Op0, Op1);
9510 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9511 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9512 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9513 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9514 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9515 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9516 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9517 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9518 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9519 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9520 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9521 case MVT::nxv2bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9522 case MVT::nxv4bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9523 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9524 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9525 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9526 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9527 default: return Register();
9528 }
9529}
9530
9531// FastEmit functions for AArch64ISD::ZIP1.
9532
9533Register fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9534 if (RetVT.SimpleTy != MVT::v8i8)
9535 return Register();
9536 if ((Subtarget->isNeonAvailable())) {
9537 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9538 }
9539 return Register();
9540}
9541
9542Register fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9543 if (RetVT.SimpleTy != MVT::v16i8)
9544 return Register();
9545 if ((Subtarget->isNeonAvailable())) {
9546 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9547 }
9548 return Register();
9549}
9550
9551Register fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9552 if (RetVT.SimpleTy != MVT::v4i16)
9553 return Register();
9554 if ((Subtarget->isNeonAvailable())) {
9555 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9556 }
9557 return Register();
9558}
9559
9560Register fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9561 if (RetVT.SimpleTy != MVT::v8i16)
9562 return Register();
9563 if ((Subtarget->isNeonAvailable())) {
9564 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9565 }
9566 return Register();
9567}
9568
9569Register fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9570 if (RetVT.SimpleTy != MVT::v2i32)
9571 return Register();
9572 if ((Subtarget->isNeonAvailable())) {
9573 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9574 }
9575 return Register();
9576}
9577
9578Register fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9579 if (RetVT.SimpleTy != MVT::v4i32)
9580 return Register();
9581 if ((Subtarget->isNeonAvailable())) {
9582 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9583 }
9584 return Register();
9585}
9586
9587Register fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9588 if (RetVT.SimpleTy != MVT::v2i64)
9589 return Register();
9590 if ((Subtarget->isNeonAvailable())) {
9591 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9592 }
9593 return Register();
9594}
9595
9596Register fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9597 if (RetVT.SimpleTy != MVT::v4f16)
9598 return Register();
9599 if ((Subtarget->isNeonAvailable())) {
9600 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9601 }
9602 return Register();
9603}
9604
9605Register fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9606 if (RetVT.SimpleTy != MVT::v8f16)
9607 return Register();
9608 if ((Subtarget->isNeonAvailable())) {
9609 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9610 }
9611 return Register();
9612}
9613
9614Register fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9615 if (RetVT.SimpleTy != MVT::v4bf16)
9616 return Register();
9617 if ((Subtarget->isNeonAvailable())) {
9618 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9619 }
9620 return Register();
9621}
9622
9623Register fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9624 if (RetVT.SimpleTy != MVT::v8bf16)
9625 return Register();
9626 if ((Subtarget->isNeonAvailable())) {
9627 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9628 }
9629 return Register();
9630}
9631
9632Register fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9633 if (RetVT.SimpleTy != MVT::v2f32)
9634 return Register();
9635 if ((Subtarget->isNeonAvailable())) {
9636 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9637 }
9638 return Register();
9639}
9640
9641Register fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9642 if (RetVT.SimpleTy != MVT::v4f32)
9643 return Register();
9644 if ((Subtarget->isNeonAvailable())) {
9645 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9646 }
9647 return Register();
9648}
9649
9650Register fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9651 if (RetVT.SimpleTy != MVT::v2f64)
9652 return Register();
9653 if ((Subtarget->isNeonAvailable())) {
9654 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9655 }
9656 return Register();
9657}
9658
9659Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9660 if (RetVT.SimpleTy != MVT::nxv2i1)
9661 return Register();
9662 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9663 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9664 }
9665 return Register();
9666}
9667
9668Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9669 if (RetVT.SimpleTy != MVT::nxv4i1)
9670 return Register();
9671 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9672 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9673 }
9674 return Register();
9675}
9676
9677Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9678 if (RetVT.SimpleTy != MVT::nxv8i1)
9679 return Register();
9680 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9681 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9682 }
9683 return Register();
9684}
9685
9686Register fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9687 if (RetVT.SimpleTy != MVT::nxv16i1)
9688 return Register();
9689 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9690 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9691 }
9692 return Register();
9693}
9694
9695Register fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9696 if (RetVT.SimpleTy != MVT::nxv16i8)
9697 return Register();
9698 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9699 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9700 }
9701 return Register();
9702}
9703
9704Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9705 if (RetVT.SimpleTy != MVT::nxv8i16)
9706 return Register();
9707 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9708 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9709 }
9710 return Register();
9711}
9712
9713Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9714 if (RetVT.SimpleTy != MVT::nxv4i32)
9715 return Register();
9716 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9717 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9718 }
9719 return Register();
9720}
9721
9722Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9723 if (RetVT.SimpleTy != MVT::nxv2i64)
9724 return Register();
9725 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9726 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9727 }
9728 return Register();
9729}
9730
9731Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9732 if (RetVT.SimpleTy != MVT::nxv2f16)
9733 return Register();
9734 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9735 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9736 }
9737 return Register();
9738}
9739
9740Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9741 if (RetVT.SimpleTy != MVT::nxv4f16)
9742 return Register();
9743 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9744 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9745 }
9746 return Register();
9747}
9748
9749Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9750 if (RetVT.SimpleTy != MVT::nxv8f16)
9751 return Register();
9752 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9753 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9754 }
9755 return Register();
9756}
9757
9758Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9759 if (RetVT.SimpleTy != MVT::nxv2bf16)
9760 return Register();
9761 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9762 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9763 }
9764 return Register();
9765}
9766
9767Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9768 if (RetVT.SimpleTy != MVT::nxv4bf16)
9769 return Register();
9770 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9771 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9772 }
9773 return Register();
9774}
9775
9776Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9777 if (RetVT.SimpleTy != MVT::nxv8bf16)
9778 return Register();
9779 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9780 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9781 }
9782 return Register();
9783}
9784
9785Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9786 if (RetVT.SimpleTy != MVT::nxv2f32)
9787 return Register();
9788 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9789 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9790 }
9791 return Register();
9792}
9793
9794Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9795 if (RetVT.SimpleTy != MVT::nxv4f32)
9796 return Register();
9797 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9798 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9799 }
9800 return Register();
9801}
9802
9803Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9804 if (RetVT.SimpleTy != MVT::nxv2f64)
9805 return Register();
9806 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9807 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9808 }
9809 return Register();
9810}
9811
9812Register fastEmit_AArch64ISD_ZIP1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9813 switch (VT.SimpleTy) {
9814 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(RetVT, Op0, Op1);
9815 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(RetVT, Op0, Op1);
9816 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(RetVT, Op0, Op1);
9817 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(RetVT, Op0, Op1);
9818 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(RetVT, Op0, Op1);
9819 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(RetVT, Op0, Op1);
9820 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(RetVT, Op0, Op1);
9821 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(RetVT, Op0, Op1);
9822 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(RetVT, Op0, Op1);
9823 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(RetVT, Op0, Op1);
9824 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(RetVT, Op0, Op1);
9825 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(RetVT, Op0, Op1);
9826 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(RetVT, Op0, Op1);
9827 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(RetVT, Op0, Op1);
9828 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9829 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9830 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9831 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9832 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9833 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9834 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9835 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9836 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9837 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9838 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9839 case MVT::nxv2bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9840 case MVT::nxv4bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9841 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9842 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9843 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9844 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9845 default: return Register();
9846 }
9847}
9848
9849// FastEmit functions for AArch64ISD::ZIP2.
9850
9851Register fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9852 if (RetVT.SimpleTy != MVT::v8i8)
9853 return Register();
9854 if ((Subtarget->isNeonAvailable())) {
9855 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9856 }
9857 return Register();
9858}
9859
9860Register fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9861 if (RetVT.SimpleTy != MVT::v16i8)
9862 return Register();
9863 if ((Subtarget->isNeonAvailable())) {
9864 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9865 }
9866 return Register();
9867}
9868
9869Register fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9870 if (RetVT.SimpleTy != MVT::v4i16)
9871 return Register();
9872 if ((Subtarget->isNeonAvailable())) {
9873 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9874 }
9875 return Register();
9876}
9877
9878Register fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9879 if (RetVT.SimpleTy != MVT::v8i16)
9880 return Register();
9881 if ((Subtarget->isNeonAvailable())) {
9882 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9883 }
9884 return Register();
9885}
9886
9887Register fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9888 if (RetVT.SimpleTy != MVT::v2i32)
9889 return Register();
9890 if ((Subtarget->isNeonAvailable())) {
9891 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9892 }
9893 return Register();
9894}
9895
9896Register fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9897 if (RetVT.SimpleTy != MVT::v4i32)
9898 return Register();
9899 if ((Subtarget->isNeonAvailable())) {
9900 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9901 }
9902 return Register();
9903}
9904
9905Register fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9906 if (RetVT.SimpleTy != MVT::v2i64)
9907 return Register();
9908 if ((Subtarget->isNeonAvailable())) {
9909 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9910 }
9911 return Register();
9912}
9913
9914Register fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9915 if (RetVT.SimpleTy != MVT::v4f16)
9916 return Register();
9917 if ((Subtarget->isNeonAvailable())) {
9918 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9919 }
9920 return Register();
9921}
9922
9923Register fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9924 if (RetVT.SimpleTy != MVT::v8f16)
9925 return Register();
9926 if ((Subtarget->isNeonAvailable())) {
9927 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9928 }
9929 return Register();
9930}
9931
9932Register fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9933 if (RetVT.SimpleTy != MVT::v4bf16)
9934 return Register();
9935 if ((Subtarget->isNeonAvailable())) {
9936 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9937 }
9938 return Register();
9939}
9940
9941Register fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9942 if (RetVT.SimpleTy != MVT::v8bf16)
9943 return Register();
9944 if ((Subtarget->isNeonAvailable())) {
9945 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9946 }
9947 return Register();
9948}
9949
9950Register fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9951 if (RetVT.SimpleTy != MVT::v2f32)
9952 return Register();
9953 if ((Subtarget->isNeonAvailable())) {
9954 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9955 }
9956 return Register();
9957}
9958
9959Register fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9960 if (RetVT.SimpleTy != MVT::v4f32)
9961 return Register();
9962 if ((Subtarget->isNeonAvailable())) {
9963 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9964 }
9965 return Register();
9966}
9967
9968Register fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9969 if (RetVT.SimpleTy != MVT::v2f64)
9970 return Register();
9971 if ((Subtarget->isNeonAvailable())) {
9972 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9973 }
9974 return Register();
9975}
9976
9977Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9978 if (RetVT.SimpleTy != MVT::nxv2i1)
9979 return Register();
9980 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9981 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9982 }
9983 return Register();
9984}
9985
9986Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9987 if (RetVT.SimpleTy != MVT::nxv4i1)
9988 return Register();
9989 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9990 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9991 }
9992 return Register();
9993}
9994
9995Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9996 if (RetVT.SimpleTy != MVT::nxv8i1)
9997 return Register();
9998 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9999 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
10000 }
10001 return Register();
10002}
10003
10004Register fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
10005 if (RetVT.SimpleTy != MVT::nxv16i1)
10006 return Register();
10007 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10008 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
10009 }
10010 return Register();
10011}
10012
10013Register fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10014 if (RetVT.SimpleTy != MVT::nxv16i8)
10015 return Register();
10016 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10017 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
10018 }
10019 return Register();
10020}
10021
10022Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10023 if (RetVT.SimpleTy != MVT::nxv8i16)
10024 return Register();
10025 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10026 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10027 }
10028 return Register();
10029}
10030
10031Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10032 if (RetVT.SimpleTy != MVT::nxv4i32)
10033 return Register();
10034 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10035 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10036 }
10037 return Register();
10038}
10039
10040Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10041 if (RetVT.SimpleTy != MVT::nxv2i64)
10042 return Register();
10043 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10044 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10045 }
10046 return Register();
10047}
10048
10049Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
10050 if (RetVT.SimpleTy != MVT::nxv2f16)
10051 return Register();
10052 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10053 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10054 }
10055 return Register();
10056}
10057
10058Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10059 if (RetVT.SimpleTy != MVT::nxv4f16)
10060 return Register();
10061 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10062 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10063 }
10064 return Register();
10065}
10066
10067Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10068 if (RetVT.SimpleTy != MVT::nxv8f16)
10069 return Register();
10070 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10071 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10072 }
10073 return Register();
10074}
10075
10076Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10077 if (RetVT.SimpleTy != MVT::nxv2bf16)
10078 return Register();
10079 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10080 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10081 }
10082 return Register();
10083}
10084
10085Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10086 if (RetVT.SimpleTy != MVT::nxv4bf16)
10087 return Register();
10088 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10089 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10090 }
10091 return Register();
10092}
10093
10094Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10095 if (RetVT.SimpleTy != MVT::nxv8bf16)
10096 return Register();
10097 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10098 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10099 }
10100 return Register();
10101}
10102
10103Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10104 if (RetVT.SimpleTy != MVT::nxv2f32)
10105 return Register();
10106 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10107 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10108 }
10109 return Register();
10110}
10111
10112Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10113 if (RetVT.SimpleTy != MVT::nxv4f32)
10114 return Register();
10115 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10116 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10117 }
10118 return Register();
10119}
10120
10121Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10122 if (RetVT.SimpleTy != MVT::nxv2f64)
10123 return Register();
10124 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10125 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10126 }
10127 return Register();
10128}
10129
10130Register fastEmit_AArch64ISD_ZIP2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10131 switch (VT.SimpleTy) {
10132 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(RetVT, Op0, Op1);
10133 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(RetVT, Op0, Op1);
10134 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(RetVT, Op0, Op1);
10135 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(RetVT, Op0, Op1);
10136 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(RetVT, Op0, Op1);
10137 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(RetVT, Op0, Op1);
10138 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(RetVT, Op0, Op1);
10139 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(RetVT, Op0, Op1);
10140 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(RetVT, Op0, Op1);
10141 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(RetVT, Op0, Op1);
10142 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(RetVT, Op0, Op1);
10143 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(RetVT, Op0, Op1);
10144 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(RetVT, Op0, Op1);
10145 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(RetVT, Op0, Op1);
10146 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
10147 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
10148 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
10149 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
10150 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10151 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10152 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10153 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10154 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
10155 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
10156 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
10157 case MVT::nxv2bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
10158 case MVT::nxv4bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
10159 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
10160 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
10161 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
10162 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
10163 default: return Register();
10164 }
10165}
10166
10167// FastEmit functions for ISD::ABDS.
10168
10169Register fastEmit_ISD_ABDS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10170 if (RetVT.SimpleTy != MVT::v8i8)
10171 return Register();
10172 if ((Subtarget->isNeonAvailable())) {
10173 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10174 }
10175 return Register();
10176}
10177
10178Register fastEmit_ISD_ABDS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10179 if (RetVT.SimpleTy != MVT::v16i8)
10180 return Register();
10181 if ((Subtarget->isNeonAvailable())) {
10182 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10183 }
10184 return Register();
10185}
10186
10187Register fastEmit_ISD_ABDS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10188 if (RetVT.SimpleTy != MVT::v4i16)
10189 return Register();
10190 if ((Subtarget->isNeonAvailable())) {
10191 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10192 }
10193 return Register();
10194}
10195
10196Register fastEmit_ISD_ABDS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10197 if (RetVT.SimpleTy != MVT::v8i16)
10198 return Register();
10199 if ((Subtarget->isNeonAvailable())) {
10200 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10201 }
10202 return Register();
10203}
10204
10205Register fastEmit_ISD_ABDS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10206 if (RetVT.SimpleTy != MVT::v2i32)
10207 return Register();
10208 if ((Subtarget->isNeonAvailable())) {
10209 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10210 }
10211 return Register();
10212}
10213
10214Register fastEmit_ISD_ABDS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10215 if (RetVT.SimpleTy != MVT::v4i32)
10216 return Register();
10217 if ((Subtarget->isNeonAvailable())) {
10218 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10219 }
10220 return Register();
10221}
10222
10223Register fastEmit_ISD_ABDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10224 switch (VT.SimpleTy) {
10225 case MVT::v8i8: return fastEmit_ISD_ABDS_MVT_v8i8_rr(RetVT, Op0, Op1);
10226 case MVT::v16i8: return fastEmit_ISD_ABDS_MVT_v16i8_rr(RetVT, Op0, Op1);
10227 case MVT::v4i16: return fastEmit_ISD_ABDS_MVT_v4i16_rr(RetVT, Op0, Op1);
10228 case MVT::v8i16: return fastEmit_ISD_ABDS_MVT_v8i16_rr(RetVT, Op0, Op1);
10229 case MVT::v2i32: return fastEmit_ISD_ABDS_MVT_v2i32_rr(RetVT, Op0, Op1);
10230 case MVT::v4i32: return fastEmit_ISD_ABDS_MVT_v4i32_rr(RetVT, Op0, Op1);
10231 default: return Register();
10232 }
10233}
10234
10235// FastEmit functions for ISD::ABDU.
10236
10237Register fastEmit_ISD_ABDU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10238 if (RetVT.SimpleTy != MVT::v8i8)
10239 return Register();
10240 if ((Subtarget->isNeonAvailable())) {
10241 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10242 }
10243 return Register();
10244}
10245
10246Register fastEmit_ISD_ABDU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10247 if (RetVT.SimpleTy != MVT::v16i8)
10248 return Register();
10249 if ((Subtarget->isNeonAvailable())) {
10250 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10251 }
10252 return Register();
10253}
10254
10255Register fastEmit_ISD_ABDU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10256 if (RetVT.SimpleTy != MVT::v4i16)
10257 return Register();
10258 if ((Subtarget->isNeonAvailable())) {
10259 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10260 }
10261 return Register();
10262}
10263
10264Register fastEmit_ISD_ABDU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10265 if (RetVT.SimpleTy != MVT::v8i16)
10266 return Register();
10267 if ((Subtarget->isNeonAvailable())) {
10268 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10269 }
10270 return Register();
10271}
10272
10273Register fastEmit_ISD_ABDU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10274 if (RetVT.SimpleTy != MVT::v2i32)
10275 return Register();
10276 if ((Subtarget->isNeonAvailable())) {
10277 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10278 }
10279 return Register();
10280}
10281
10282Register fastEmit_ISD_ABDU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10283 if (RetVT.SimpleTy != MVT::v4i32)
10284 return Register();
10285 if ((Subtarget->isNeonAvailable())) {
10286 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10287 }
10288 return Register();
10289}
10290
10291Register fastEmit_ISD_ABDU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10292 switch (VT.SimpleTy) {
10293 case MVT::v8i8: return fastEmit_ISD_ABDU_MVT_v8i8_rr(RetVT, Op0, Op1);
10294 case MVT::v16i8: return fastEmit_ISD_ABDU_MVT_v16i8_rr(RetVT, Op0, Op1);
10295 case MVT::v4i16: return fastEmit_ISD_ABDU_MVT_v4i16_rr(RetVT, Op0, Op1);
10296 case MVT::v8i16: return fastEmit_ISD_ABDU_MVT_v8i16_rr(RetVT, Op0, Op1);
10297 case MVT::v2i32: return fastEmit_ISD_ABDU_MVT_v2i32_rr(RetVT, Op0, Op1);
10298 case MVT::v4i32: return fastEmit_ISD_ABDU_MVT_v4i32_rr(RetVT, Op0, Op1);
10299 default: return Register();
10300 }
10301}
10302
10303// FastEmit functions for ISD::ADD.
10304
10305Register fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10306 if (RetVT.SimpleTy != MVT::i32)
10307 return Register();
10308 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
10309}
10310
10311Register fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10312 if (RetVT.SimpleTy != MVT::i64)
10313 return Register();
10314 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
10315}
10316
10317Register fastEmit_ISD_ADD_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10318 if (RetVT.SimpleTy != MVT::v8i8)
10319 return Register();
10320 if ((Subtarget->isNeonAvailable())) {
10321 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10322 }
10323 return Register();
10324}
10325
10326Register fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10327 if (RetVT.SimpleTy != MVT::v16i8)
10328 return Register();
10329 if ((Subtarget->isNeonAvailable())) {
10330 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10331 }
10332 return Register();
10333}
10334
10335Register fastEmit_ISD_ADD_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10336 if (RetVT.SimpleTy != MVT::v4i16)
10337 return Register();
10338 if ((Subtarget->isNeonAvailable())) {
10339 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10340 }
10341 return Register();
10342}
10343
10344Register fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10345 if (RetVT.SimpleTy != MVT::v8i16)
10346 return Register();
10347 if ((Subtarget->isNeonAvailable())) {
10348 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10349 }
10350 return Register();
10351}
10352
10353Register fastEmit_ISD_ADD_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10354 if (RetVT.SimpleTy != MVT::v2i32)
10355 return Register();
10356 if ((Subtarget->isNeonAvailable())) {
10357 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10358 }
10359 return Register();
10360}
10361
10362Register fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10363 if (RetVT.SimpleTy != MVT::v4i32)
10364 return Register();
10365 if ((Subtarget->isNeonAvailable())) {
10366 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10367 }
10368 return Register();
10369}
10370
10371Register fastEmit_ISD_ADD_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
10372 if (RetVT.SimpleTy != MVT::v1i64)
10373 return Register();
10374 if ((Subtarget->isNeonAvailable())) {
10375 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
10376 }
10377 return Register();
10378}
10379
10380Register fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10381 if (RetVT.SimpleTy != MVT::v2i64)
10382 return Register();
10383 if ((Subtarget->isNeonAvailable())) {
10384 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10385 }
10386 return Register();
10387}
10388
10389Register fastEmit_ISD_ADD_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10390 if (RetVT.SimpleTy != MVT::nxv16i8)
10391 return Register();
10392 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10393 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
10394 }
10395 return Register();
10396}
10397
10398Register fastEmit_ISD_ADD_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10399 if (RetVT.SimpleTy != MVT::nxv8i16)
10400 return Register();
10401 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10402 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10403 }
10404 return Register();
10405}
10406
10407Register fastEmit_ISD_ADD_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10408 if (RetVT.SimpleTy != MVT::nxv4i32)
10409 return Register();
10410 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10411 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10412 }
10413 return Register();
10414}
10415
10416Register fastEmit_ISD_ADD_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10417 if (RetVT.SimpleTy != MVT::nxv2i64)
10418 return Register();
10419 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10420 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10421 }
10422 return Register();
10423}
10424
10425Register fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10426 switch (VT.SimpleTy) {
10427 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
10428 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op1);
10429 case MVT::v8i8: return fastEmit_ISD_ADD_MVT_v8i8_rr(RetVT, Op0, Op1);
10430 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
10431 case MVT::v4i16: return fastEmit_ISD_ADD_MVT_v4i16_rr(RetVT, Op0, Op1);
10432 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
10433 case MVT::v2i32: return fastEmit_ISD_ADD_MVT_v2i32_rr(RetVT, Op0, Op1);
10434 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
10435 case MVT::v1i64: return fastEmit_ISD_ADD_MVT_v1i64_rr(RetVT, Op0, Op1);
10436 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
10437 case MVT::nxv16i8: return fastEmit_ISD_ADD_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10438 case MVT::nxv8i16: return fastEmit_ISD_ADD_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10439 case MVT::nxv4i32: return fastEmit_ISD_ADD_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10440 case MVT::nxv2i64: return fastEmit_ISD_ADD_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10441 default: return Register();
10442 }
10443}
10444
10445// FastEmit functions for ISD::AND.
10446
10447Register fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10448 if (RetVT.SimpleTy != MVT::i32)
10449 return Register();
10450 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
10451}
10452
10453Register fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10454 if (RetVT.SimpleTy != MVT::i64)
10455 return Register();
10456 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
10457}
10458
10459Register fastEmit_ISD_AND_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10460 if (RetVT.SimpleTy != MVT::v8i8)
10461 return Register();
10462 if ((Subtarget->isNeonAvailable())) {
10463 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10464 }
10465 return Register();
10466}
10467
10468Register fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10469 if (RetVT.SimpleTy != MVT::v16i8)
10470 return Register();
10471 if ((Subtarget->isNeonAvailable())) {
10472 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10473 }
10474 return Register();
10475}
10476
10477Register fastEmit_ISD_AND_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10478 if (RetVT.SimpleTy != MVT::v4i16)
10479 return Register();
10480 if ((Subtarget->isNeonAvailable())) {
10481 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10482 }
10483 return Register();
10484}
10485
10486Register fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10487 if (RetVT.SimpleTy != MVT::v8i16)
10488 return Register();
10489 if ((Subtarget->isNeonAvailable())) {
10490 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10491 }
10492 return Register();
10493}
10494
10495Register fastEmit_ISD_AND_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10496 if (RetVT.SimpleTy != MVT::v2i32)
10497 return Register();
10498 if ((Subtarget->isNeonAvailable())) {
10499 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10500 }
10501 return Register();
10502}
10503
10504Register fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10505 if (RetVT.SimpleTy != MVT::v4i32)
10506 return Register();
10507 if ((Subtarget->isNeonAvailable())) {
10508 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10509 }
10510 return Register();
10511}
10512
10513Register fastEmit_ISD_AND_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
10514 if (RetVT.SimpleTy != MVT::v1i64)
10515 return Register();
10516 if ((Subtarget->isNeonAvailable())) {
10517 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10518 }
10519 return Register();
10520}
10521
10522Register fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10523 if (RetVT.SimpleTy != MVT::v2i64)
10524 return Register();
10525 if ((Subtarget->isNeonAvailable())) {
10526 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10527 }
10528 return Register();
10529}
10530
10531Register fastEmit_ISD_AND_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10532 if (RetVT.SimpleTy != MVT::nxv16i8)
10533 return Register();
10534 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10535 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10536 }
10537 return Register();
10538}
10539
10540Register fastEmit_ISD_AND_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10541 if (RetVT.SimpleTy != MVT::nxv8i16)
10542 return Register();
10543 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10544 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10545 }
10546 return Register();
10547}
10548
10549Register fastEmit_ISD_AND_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10550 if (RetVT.SimpleTy != MVT::nxv4i32)
10551 return Register();
10552 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10553 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10554 }
10555 return Register();
10556}
10557
10558Register fastEmit_ISD_AND_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10559 if (RetVT.SimpleTy != MVT::nxv2i64)
10560 return Register();
10561 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10562 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10563 }
10564 return Register();
10565}
10566
10567Register fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10568 switch (VT.SimpleTy) {
10569 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
10570 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op1);
10571 case MVT::v8i8: return fastEmit_ISD_AND_MVT_v8i8_rr(RetVT, Op0, Op1);
10572 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
10573 case MVT::v4i16: return fastEmit_ISD_AND_MVT_v4i16_rr(RetVT, Op0, Op1);
10574 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
10575 case MVT::v2i32: return fastEmit_ISD_AND_MVT_v2i32_rr(RetVT, Op0, Op1);
10576 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
10577 case MVT::v1i64: return fastEmit_ISD_AND_MVT_v1i64_rr(RetVT, Op0, Op1);
10578 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
10579 case MVT::nxv16i8: return fastEmit_ISD_AND_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10580 case MVT::nxv8i16: return fastEmit_ISD_AND_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10581 case MVT::nxv4i32: return fastEmit_ISD_AND_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10582 case MVT::nxv2i64: return fastEmit_ISD_AND_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10583 default: return Register();
10584 }
10585}
10586
10587// FastEmit functions for ISD::AVGCEILS.
10588
10589Register fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10590 if (RetVT.SimpleTy != MVT::v8i8)
10591 return Register();
10592 if ((Subtarget->isNeonAvailable())) {
10593 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10594 }
10595 return Register();
10596}
10597
10598Register fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10599 if (RetVT.SimpleTy != MVT::v16i8)
10600 return Register();
10601 if ((Subtarget->isNeonAvailable())) {
10602 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10603 }
10604 return Register();
10605}
10606
10607Register fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10608 if (RetVT.SimpleTy != MVT::v4i16)
10609 return Register();
10610 if ((Subtarget->isNeonAvailable())) {
10611 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10612 }
10613 return Register();
10614}
10615
10616Register fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10617 if (RetVT.SimpleTy != MVT::v8i16)
10618 return Register();
10619 if ((Subtarget->isNeonAvailable())) {
10620 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10621 }
10622 return Register();
10623}
10624
10625Register fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10626 if (RetVT.SimpleTy != MVT::v2i32)
10627 return Register();
10628 if ((Subtarget->isNeonAvailable())) {
10629 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10630 }
10631 return Register();
10632}
10633
10634Register fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10635 if (RetVT.SimpleTy != MVT::v4i32)
10636 return Register();
10637 if ((Subtarget->isNeonAvailable())) {
10638 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10639 }
10640 return Register();
10641}
10642
10643Register fastEmit_ISD_AVGCEILS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10644 switch (VT.SimpleTy) {
10645 case MVT::v8i8: return fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(RetVT, Op0, Op1);
10646 case MVT::v16i8: return fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(RetVT, Op0, Op1);
10647 case MVT::v4i16: return fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(RetVT, Op0, Op1);
10648 case MVT::v8i16: return fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(RetVT, Op0, Op1);
10649 case MVT::v2i32: return fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(RetVT, Op0, Op1);
10650 case MVT::v4i32: return fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(RetVT, Op0, Op1);
10651 default: return Register();
10652 }
10653}
10654
10655// FastEmit functions for ISD::AVGCEILU.
10656
10657Register fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10658 if (RetVT.SimpleTy != MVT::v8i8)
10659 return Register();
10660 if ((Subtarget->isNeonAvailable())) {
10661 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10662 }
10663 return Register();
10664}
10665
10666Register fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10667 if (RetVT.SimpleTy != MVT::v16i8)
10668 return Register();
10669 if ((Subtarget->isNeonAvailable())) {
10670 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10671 }
10672 return Register();
10673}
10674
10675Register fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10676 if (RetVT.SimpleTy != MVT::v4i16)
10677 return Register();
10678 if ((Subtarget->isNeonAvailable())) {
10679 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10680 }
10681 return Register();
10682}
10683
10684Register fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10685 if (RetVT.SimpleTy != MVT::v8i16)
10686 return Register();
10687 if ((Subtarget->isNeonAvailable())) {
10688 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10689 }
10690 return Register();
10691}
10692
10693Register fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10694 if (RetVT.SimpleTy != MVT::v2i32)
10695 return Register();
10696 if ((Subtarget->isNeonAvailable())) {
10697 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10698 }
10699 return Register();
10700}
10701
10702Register fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10703 if (RetVT.SimpleTy != MVT::v4i32)
10704 return Register();
10705 if ((Subtarget->isNeonAvailable())) {
10706 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10707 }
10708 return Register();
10709}
10710
10711Register fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10712 switch (VT.SimpleTy) {
10713 case MVT::v8i8: return fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(RetVT, Op0, Op1);
10714 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
10715 case MVT::v4i16: return fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(RetVT, Op0, Op1);
10716 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
10717 case MVT::v2i32: return fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(RetVT, Op0, Op1);
10718 case MVT::v4i32: return fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(RetVT, Op0, Op1);
10719 default: return Register();
10720 }
10721}
10722
10723// FastEmit functions for ISD::AVGFLOORS.
10724
10725Register fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10726 if (RetVT.SimpleTy != MVT::v8i8)
10727 return Register();
10728 if ((Subtarget->isNeonAvailable())) {
10729 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10730 }
10731 return Register();
10732}
10733
10734Register fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10735 if (RetVT.SimpleTy != MVT::v16i8)
10736 return Register();
10737 if ((Subtarget->isNeonAvailable())) {
10738 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10739 }
10740 return Register();
10741}
10742
10743Register fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10744 if (RetVT.SimpleTy != MVT::v4i16)
10745 return Register();
10746 if ((Subtarget->isNeonAvailable())) {
10747 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10748 }
10749 return Register();
10750}
10751
10752Register fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10753 if (RetVT.SimpleTy != MVT::v8i16)
10754 return Register();
10755 if ((Subtarget->isNeonAvailable())) {
10756 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10757 }
10758 return Register();
10759}
10760
10761Register fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10762 if (RetVT.SimpleTy != MVT::v2i32)
10763 return Register();
10764 if ((Subtarget->isNeonAvailable())) {
10765 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10766 }
10767 return Register();
10768}
10769
10770Register fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10771 if (RetVT.SimpleTy != MVT::v4i32)
10772 return Register();
10773 if ((Subtarget->isNeonAvailable())) {
10774 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10775 }
10776 return Register();
10777}
10778
10779Register fastEmit_ISD_AVGFLOORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10780 switch (VT.SimpleTy) {
10781 case MVT::v8i8: return fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(RetVT, Op0, Op1);
10782 case MVT::v16i8: return fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(RetVT, Op0, Op1);
10783 case MVT::v4i16: return fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(RetVT, Op0, Op1);
10784 case MVT::v8i16: return fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(RetVT, Op0, Op1);
10785 case MVT::v2i32: return fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(RetVT, Op0, Op1);
10786 case MVT::v4i32: return fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(RetVT, Op0, Op1);
10787 default: return Register();
10788 }
10789}
10790
10791// FastEmit functions for ISD::AVGFLOORU.
10792
10793Register fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10794 if (RetVT.SimpleTy != MVT::v8i8)
10795 return Register();
10796 if ((Subtarget->isNeonAvailable())) {
10797 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10798 }
10799 return Register();
10800}
10801
10802Register fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10803 if (RetVT.SimpleTy != MVT::v16i8)
10804 return Register();
10805 if ((Subtarget->isNeonAvailable())) {
10806 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10807 }
10808 return Register();
10809}
10810
10811Register fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10812 if (RetVT.SimpleTy != MVT::v4i16)
10813 return Register();
10814 if ((Subtarget->isNeonAvailable())) {
10815 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10816 }
10817 return Register();
10818}
10819
10820Register fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10821 if (RetVT.SimpleTy != MVT::v8i16)
10822 return Register();
10823 if ((Subtarget->isNeonAvailable())) {
10824 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10825 }
10826 return Register();
10827}
10828
10829Register fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10830 if (RetVT.SimpleTy != MVT::v2i32)
10831 return Register();
10832 if ((Subtarget->isNeonAvailable())) {
10833 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10834 }
10835 return Register();
10836}
10837
10838Register fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10839 if (RetVT.SimpleTy != MVT::v4i32)
10840 return Register();
10841 if ((Subtarget->isNeonAvailable())) {
10842 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10843 }
10844 return Register();
10845}
10846
10847Register fastEmit_ISD_AVGFLOORU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10848 switch (VT.SimpleTy) {
10849 case MVT::v8i8: return fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(RetVT, Op0, Op1);
10850 case MVT::v16i8: return fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(RetVT, Op0, Op1);
10851 case MVT::v4i16: return fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(RetVT, Op0, Op1);
10852 case MVT::v8i16: return fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(RetVT, Op0, Op1);
10853 case MVT::v2i32: return fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(RetVT, Op0, Op1);
10854 case MVT::v4i32: return fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(RetVT, Op0, Op1);
10855 default: return Register();
10856 }
10857}
10858
10859// FastEmit functions for ISD::CLMUL.
10860
10861Register fastEmit_ISD_CLMUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10862 if (RetVT.SimpleTy != MVT::v8i8)
10863 return Register();
10864 if ((Subtarget->isNeonAvailable())) {
10865 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10866 }
10867 return Register();
10868}
10869
10870Register fastEmit_ISD_CLMUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10871 if (RetVT.SimpleTy != MVT::v16i8)
10872 return Register();
10873 if ((Subtarget->isNeonAvailable())) {
10874 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10875 }
10876 return Register();
10877}
10878
10879Register fastEmit_ISD_CLMUL_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10880 if (RetVT.SimpleTy != MVT::nxv16i8)
10881 return Register();
10882 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2() || Subtarget->hasSME()))) {
10883 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMUL_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
10884 }
10885 return Register();
10886}
10887
10888Register fastEmit_ISD_CLMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10889 switch (VT.SimpleTy) {
10890 case MVT::v8i8: return fastEmit_ISD_CLMUL_MVT_v8i8_rr(RetVT, Op0, Op1);
10891 case MVT::v16i8: return fastEmit_ISD_CLMUL_MVT_v16i8_rr(RetVT, Op0, Op1);
10892 case MVT::nxv16i8: return fastEmit_ISD_CLMUL_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10893 default: return Register();
10894 }
10895}
10896
10897// FastEmit functions for ISD::CONCAT_VECTORS.
10898
10899Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(MVT RetVT, Register Op0, Register Op1) {
10900 if (RetVT.SimpleTy != MVT::nxv2i1)
10901 return Register();
10902 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10903 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
10904 }
10905 return Register();
10906}
10907
10908Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
10909 if (RetVT.SimpleTy != MVT::nxv4i1)
10910 return Register();
10911 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10912 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
10913 }
10914 return Register();
10915}
10916
10917Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
10918 if (RetVT.SimpleTy != MVT::nxv8i1)
10919 return Register();
10920 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10921 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
10922 }
10923 return Register();
10924}
10925
10926Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
10927 if (RetVT.SimpleTy != MVT::nxv16i1)
10928 return Register();
10929 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10930 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
10931 }
10932 return Register();
10933}
10934
10935Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
10936 if (RetVT.SimpleTy != MVT::nxv4f16)
10937 return Register();
10938 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10939 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10940 }
10941 return Register();
10942}
10943
10944Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10945 if (RetVT.SimpleTy != MVT::nxv8f16)
10946 return Register();
10947 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10948 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10949 }
10950 return Register();
10951}
10952
10953Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10954 if (RetVT.SimpleTy != MVT::nxv4bf16)
10955 return Register();
10956 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10957 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10958 }
10959 return Register();
10960}
10961
10962Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10963 if (RetVT.SimpleTy != MVT::nxv8bf16)
10964 return Register();
10965 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10966 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10967 }
10968 return Register();
10969}
10970
10971Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10972 if (RetVT.SimpleTy != MVT::nxv4f32)
10973 return Register();
10974 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10975 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10976 }
10977 return Register();
10978}
10979
10980Register fastEmit_ISD_CONCAT_VECTORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10981 switch (VT.SimpleTy) {
10982 case MVT::nxv1i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(RetVT, Op0, Op1);
10983 case MVT::nxv2i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(RetVT, Op0, Op1);
10984 case MVT::nxv4i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(RetVT, Op0, Op1);
10985 case MVT::nxv8i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(RetVT, Op0, Op1);
10986 case MVT::nxv2f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(RetVT, Op0, Op1);
10987 case MVT::nxv4f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(RetVT, Op0, Op1);
10988 case MVT::nxv2bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
10989 case MVT::nxv4bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
10990 case MVT::nxv2f32: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(RetVT, Op0, Op1);
10991 default: return Register();
10992 }
10993}
10994
10995// FastEmit functions for ISD::FADD.
10996
10997Register fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10998 if (RetVT.SimpleTy != MVT::f16)
10999 return Register();
11000 if ((Subtarget->hasFullFP16())) {
11001 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11002 }
11003 return Register();
11004}
11005
11006Register fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11007 if (RetVT.SimpleTy != MVT::f32)
11008 return Register();
11009 if ((Subtarget->hasFPARMv8())) {
11010 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11011 }
11012 return Register();
11013}
11014
11015Register fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11016 if (RetVT.SimpleTy != MVT::f64)
11017 return Register();
11018 if ((Subtarget->hasFPARMv8())) {
11019 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11020 }
11021 return Register();
11022}
11023
11024Register fastEmit_ISD_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11025 if (RetVT.SimpleTy != MVT::v4f16)
11026 return Register();
11027 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11028 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11029 }
11030 return Register();
11031}
11032
11033Register fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11034 if (RetVT.SimpleTy != MVT::v8f16)
11035 return Register();
11036 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11037 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11038 }
11039 return Register();
11040}
11041
11042Register fastEmit_ISD_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11043 if (RetVT.SimpleTy != MVT::v2f32)
11044 return Register();
11045 if ((Subtarget->isNeonAvailable())) {
11046 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11047 }
11048 return Register();
11049}
11050
11051Register fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11052 if (RetVT.SimpleTy != MVT::v4f32)
11053 return Register();
11054 if ((Subtarget->isNeonAvailable())) {
11055 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11056 }
11057 return Register();
11058}
11059
11060Register fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11061 if (RetVT.SimpleTy != MVT::v2f64)
11062 return Register();
11063 if ((Subtarget->isNeonAvailable())) {
11064 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11065 }
11066 return Register();
11067}
11068
11069Register fastEmit_ISD_FADD_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11070 if (RetVT.SimpleTy != MVT::nxv8f16)
11071 return Register();
11072 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11073 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11074 }
11075 return Register();
11076}
11077
11078Register fastEmit_ISD_FADD_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11079 if (RetVT.SimpleTy != MVT::nxv8bf16)
11080 return Register();
11081 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
11082 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFADD_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11083 }
11084 return Register();
11085}
11086
11087Register fastEmit_ISD_FADD_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11088 if (RetVT.SimpleTy != MVT::nxv4f32)
11089 return Register();
11090 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11091 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11092 }
11093 return Register();
11094}
11095
11096Register fastEmit_ISD_FADD_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11097 if (RetVT.SimpleTy != MVT::nxv2f64)
11098 return Register();
11099 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11100 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11101 }
11102 return Register();
11103}
11104
11105Register fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11106 switch (VT.SimpleTy) {
11107 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
11108 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
11109 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
11110 case MVT::v4f16: return fastEmit_ISD_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
11111 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
11112 case MVT::v2f32: return fastEmit_ISD_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
11113 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
11114 case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
11115 case MVT::nxv8f16: return fastEmit_ISD_FADD_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11116 case MVT::nxv8bf16: return fastEmit_ISD_FADD_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11117 case MVT::nxv4f32: return fastEmit_ISD_FADD_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11118 case MVT::nxv2f64: return fastEmit_ISD_FADD_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11119 default: return Register();
11120 }
11121}
11122
11123// FastEmit functions for ISD::FDIV.
11124
11125Register fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11126 if (RetVT.SimpleTy != MVT::f16)
11127 return Register();
11128 if ((Subtarget->hasFullFP16())) {
11129 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11130 }
11131 return Register();
11132}
11133
11134Register fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11135 if (RetVT.SimpleTy != MVT::f32)
11136 return Register();
11137 if ((Subtarget->hasFPARMv8())) {
11138 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11139 }
11140 return Register();
11141}
11142
11143Register fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11144 if (RetVT.SimpleTy != MVT::f64)
11145 return Register();
11146 if ((Subtarget->hasFPARMv8())) {
11147 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11148 }
11149 return Register();
11150}
11151
11152Register fastEmit_ISD_FDIV_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11153 if (RetVT.SimpleTy != MVT::v4f16)
11154 return Register();
11155 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11156 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11157 }
11158 return Register();
11159}
11160
11161Register fastEmit_ISD_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11162 if (RetVT.SimpleTy != MVT::v8f16)
11163 return Register();
11164 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11165 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11166 }
11167 return Register();
11168}
11169
11170Register fastEmit_ISD_FDIV_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11171 if (RetVT.SimpleTy != MVT::v2f32)
11172 return Register();
11173 if ((Subtarget->isNeonAvailable())) {
11174 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11175 }
11176 return Register();
11177}
11178
11179Register fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11180 if (RetVT.SimpleTy != MVT::v4f32)
11181 return Register();
11182 if ((Subtarget->isNeonAvailable())) {
11183 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11184 }
11185 return Register();
11186}
11187
11188Register fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11189 if (RetVT.SimpleTy != MVT::v2f64)
11190 return Register();
11191 if ((Subtarget->isNeonAvailable())) {
11192 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11193 }
11194 return Register();
11195}
11196
11197Register fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11198 switch (VT.SimpleTy) {
11199 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
11200 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
11201 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
11202 case MVT::v4f16: return fastEmit_ISD_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1);
11203 case MVT::v8f16: return fastEmit_ISD_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
11204 case MVT::v2f32: return fastEmit_ISD_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1);
11205 case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
11206 case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
11207 default: return Register();
11208 }
11209}
11210
11211// FastEmit functions for ISD::FMAXIMUM.
11212
11213Register fastEmit_ISD_FMAXIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11214 if (RetVT.SimpleTy != MVT::f16)
11215 return Register();
11216 if ((Subtarget->hasFullFP16())) {
11217 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11218 }
11219 return Register();
11220}
11221
11222Register fastEmit_ISD_FMAXIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11223 if (RetVT.SimpleTy != MVT::f32)
11224 return Register();
11225 if ((Subtarget->hasFPARMv8())) {
11226 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11227 }
11228 return Register();
11229}
11230
11231Register fastEmit_ISD_FMAXIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11232 if (RetVT.SimpleTy != MVT::f64)
11233 return Register();
11234 if ((Subtarget->hasFPARMv8())) {
11235 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11236 }
11237 return Register();
11238}
11239
11240Register fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11241 if (RetVT.SimpleTy != MVT::v4f16)
11242 return Register();
11243 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11244 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11245 }
11246 return Register();
11247}
11248
11249Register fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11250 if (RetVT.SimpleTy != MVT::v8f16)
11251 return Register();
11252 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11253 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11254 }
11255 return Register();
11256}
11257
11258Register fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11259 if (RetVT.SimpleTy != MVT::v2f32)
11260 return Register();
11261 if ((Subtarget->isNeonAvailable())) {
11262 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11263 }
11264 return Register();
11265}
11266
11267Register fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11268 if (RetVT.SimpleTy != MVT::v4f32)
11269 return Register();
11270 if ((Subtarget->isNeonAvailable())) {
11271 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11272 }
11273 return Register();
11274}
11275
11276Register fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11277 if (RetVT.SimpleTy != MVT::v1f64)
11278 return Register();
11279 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11280}
11281
11282Register fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11283 if (RetVT.SimpleTy != MVT::v2f64)
11284 return Register();
11285 if ((Subtarget->isNeonAvailable())) {
11286 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11287 }
11288 return Register();
11289}
11290
11291Register fastEmit_ISD_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11292 switch (VT.SimpleTy) {
11293 case MVT::f16: return fastEmit_ISD_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1);
11294 case MVT::f32: return fastEmit_ISD_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1);
11295 case MVT::f64: return fastEmit_ISD_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1);
11296 case MVT::v4f16: return fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11297 case MVT::v8f16: return fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11298 case MVT::v2f32: return fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11299 case MVT::v4f32: return fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11300 case MVT::v1f64: return fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11301 case MVT::v2f64: return fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11302 default: return Register();
11303 }
11304}
11305
11306// FastEmit functions for ISD::FMAXNUM.
11307
11308Register fastEmit_ISD_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11309 if (RetVT.SimpleTy != MVT::f16)
11310 return Register();
11311 if ((Subtarget->hasFullFP16())) {
11312 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11313 }
11314 return Register();
11315}
11316
11317Register fastEmit_ISD_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11318 if (RetVT.SimpleTy != MVT::f32)
11319 return Register();
11320 if ((Subtarget->hasFPARMv8())) {
11321 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11322 }
11323 return Register();
11324}
11325
11326Register fastEmit_ISD_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11327 if (RetVT.SimpleTy != MVT::f64)
11328 return Register();
11329 if ((Subtarget->hasFPARMv8())) {
11330 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11331 }
11332 return Register();
11333}
11334
11335Register fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11336 if (RetVT.SimpleTy != MVT::v4f16)
11337 return Register();
11338 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11339 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11340 }
11341 return Register();
11342}
11343
11344Register fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11345 if (RetVT.SimpleTy != MVT::v8f16)
11346 return Register();
11347 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11348 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11349 }
11350 return Register();
11351}
11352
11353Register fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11354 if (RetVT.SimpleTy != MVT::v2f32)
11355 return Register();
11356 if ((Subtarget->isNeonAvailable())) {
11357 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11358 }
11359 return Register();
11360}
11361
11362Register fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11363 if (RetVT.SimpleTy != MVT::v4f32)
11364 return Register();
11365 if ((Subtarget->isNeonAvailable())) {
11366 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11367 }
11368 return Register();
11369}
11370
11371Register fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11372 if (RetVT.SimpleTy != MVT::v1f64)
11373 return Register();
11374 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11375}
11376
11377Register fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11378 if (RetVT.SimpleTy != MVT::v2f64)
11379 return Register();
11380 if ((Subtarget->isNeonAvailable())) {
11381 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11382 }
11383 return Register();
11384}
11385
11386Register fastEmit_ISD_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11387 switch (VT.SimpleTy) {
11388 case MVT::f16: return fastEmit_ISD_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
11389 case MVT::f32: return fastEmit_ISD_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
11390 case MVT::f64: return fastEmit_ISD_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
11391 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11392 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11393 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11394 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11395 case MVT::v1f64: return fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11396 case MVT::v2f64: return fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11397 default: return Register();
11398 }
11399}
11400
11401// FastEmit functions for ISD::FMAXNUM_IEEE.
11402
11403Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11404 if (RetVT.SimpleTy != MVT::f16)
11405 return Register();
11406 if ((Subtarget->hasFullFP16())) {
11407 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11408 }
11409 return Register();
11410}
11411
11412Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11413 if (RetVT.SimpleTy != MVT::f32)
11414 return Register();
11415 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11416}
11417
11418Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11419 if (RetVT.SimpleTy != MVT::f64)
11420 return Register();
11421 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11422}
11423
11424Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11425 if (RetVT.SimpleTy != MVT::v4f16)
11426 return Register();
11427 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11428 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11429 }
11430 return Register();
11431}
11432
11433Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11434 if (RetVT.SimpleTy != MVT::v8f16)
11435 return Register();
11436 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11437 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11438 }
11439 return Register();
11440}
11441
11442Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11443 if (RetVT.SimpleTy != MVT::v2f32)
11444 return Register();
11445 if ((Subtarget->isNeonAvailable())) {
11446 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11447 }
11448 return Register();
11449}
11450
11451Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11452 if (RetVT.SimpleTy != MVT::v4f32)
11453 return Register();
11454 if ((Subtarget->isNeonAvailable())) {
11455 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11456 }
11457 return Register();
11458}
11459
11460Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11461 if (RetVT.SimpleTy != MVT::v2f64)
11462 return Register();
11463 if ((Subtarget->isNeonAvailable())) {
11464 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11465 }
11466 return Register();
11467}
11468
11469Register fastEmit_ISD_FMAXNUM_IEEE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11470 switch (VT.SimpleTy) {
11471 case MVT::f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f16_rr(RetVT, Op0, Op1);
11472 case MVT::f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f32_rr(RetVT, Op0, Op1);
11473 case MVT::f64: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f64_rr(RetVT, Op0, Op1);
11474 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f16_rr(RetVT, Op0, Op1);
11475 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v8f16_rr(RetVT, Op0, Op1);
11476 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f32_rr(RetVT, Op0, Op1);
11477 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f32_rr(RetVT, Op0, Op1);
11478 case MVT::v2f64: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f64_rr(RetVT, Op0, Op1);
11479 default: return Register();
11480 }
11481}
11482
11483// FastEmit functions for ISD::FMINIMUM.
11484
11485Register fastEmit_ISD_FMINIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11486 if (RetVT.SimpleTy != MVT::f16)
11487 return Register();
11488 if ((Subtarget->hasFullFP16())) {
11489 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11490 }
11491 return Register();
11492}
11493
11494Register fastEmit_ISD_FMINIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11495 if (RetVT.SimpleTy != MVT::f32)
11496 return Register();
11497 if ((Subtarget->hasFPARMv8())) {
11498 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11499 }
11500 return Register();
11501}
11502
11503Register fastEmit_ISD_FMINIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11504 if (RetVT.SimpleTy != MVT::f64)
11505 return Register();
11506 if ((Subtarget->hasFPARMv8())) {
11507 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11508 }
11509 return Register();
11510}
11511
11512Register fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11513 if (RetVT.SimpleTy != MVT::v4f16)
11514 return Register();
11515 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11516 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11517 }
11518 return Register();
11519}
11520
11521Register fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11522 if (RetVT.SimpleTy != MVT::v8f16)
11523 return Register();
11524 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11525 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11526 }
11527 return Register();
11528}
11529
11530Register fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11531 if (RetVT.SimpleTy != MVT::v2f32)
11532 return Register();
11533 if ((Subtarget->isNeonAvailable())) {
11534 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11535 }
11536 return Register();
11537}
11538
11539Register fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11540 if (RetVT.SimpleTy != MVT::v4f32)
11541 return Register();
11542 if ((Subtarget->isNeonAvailable())) {
11543 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11544 }
11545 return Register();
11546}
11547
11548Register fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11549 if (RetVT.SimpleTy != MVT::v1f64)
11550 return Register();
11551 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11552}
11553
11554Register fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11555 if (RetVT.SimpleTy != MVT::v2f64)
11556 return Register();
11557 if ((Subtarget->isNeonAvailable())) {
11558 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11559 }
11560 return Register();
11561}
11562
11563Register fastEmit_ISD_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11564 switch (VT.SimpleTy) {
11565 case MVT::f16: return fastEmit_ISD_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1);
11566 case MVT::f32: return fastEmit_ISD_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1);
11567 case MVT::f64: return fastEmit_ISD_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1);
11568 case MVT::v4f16: return fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11569 case MVT::v8f16: return fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11570 case MVT::v2f32: return fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11571 case MVT::v4f32: return fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11572 case MVT::v1f64: return fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11573 case MVT::v2f64: return fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11574 default: return Register();
11575 }
11576}
11577
11578// FastEmit functions for ISD::FMINNUM.
11579
11580Register fastEmit_ISD_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11581 if (RetVT.SimpleTy != MVT::f16)
11582 return Register();
11583 if ((Subtarget->hasFullFP16())) {
11584 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11585 }
11586 return Register();
11587}
11588
11589Register fastEmit_ISD_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11590 if (RetVT.SimpleTy != MVT::f32)
11591 return Register();
11592 if ((Subtarget->hasFPARMv8())) {
11593 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11594 }
11595 return Register();
11596}
11597
11598Register fastEmit_ISD_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11599 if (RetVT.SimpleTy != MVT::f64)
11600 return Register();
11601 if ((Subtarget->hasFPARMv8())) {
11602 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11603 }
11604 return Register();
11605}
11606
11607Register fastEmit_ISD_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11608 if (RetVT.SimpleTy != MVT::v4f16)
11609 return Register();
11610 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11611 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11612 }
11613 return Register();
11614}
11615
11616Register fastEmit_ISD_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11617 if (RetVT.SimpleTy != MVT::v8f16)
11618 return Register();
11619 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11620 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11621 }
11622 return Register();
11623}
11624
11625Register fastEmit_ISD_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11626 if (RetVT.SimpleTy != MVT::v2f32)
11627 return Register();
11628 if ((Subtarget->isNeonAvailable())) {
11629 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11630 }
11631 return Register();
11632}
11633
11634Register fastEmit_ISD_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11635 if (RetVT.SimpleTy != MVT::v4f32)
11636 return Register();
11637 if ((Subtarget->isNeonAvailable())) {
11638 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11639 }
11640 return Register();
11641}
11642
11643Register fastEmit_ISD_FMINNUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11644 if (RetVT.SimpleTy != MVT::v1f64)
11645 return Register();
11646 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11647}
11648
11649Register fastEmit_ISD_FMINNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11650 if (RetVT.SimpleTy != MVT::v2f64)
11651 return Register();
11652 if ((Subtarget->isNeonAvailable())) {
11653 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11654 }
11655 return Register();
11656}
11657
11658Register fastEmit_ISD_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11659 switch (VT.SimpleTy) {
11660 case MVT::f16: return fastEmit_ISD_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
11661 case MVT::f32: return fastEmit_ISD_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
11662 case MVT::f64: return fastEmit_ISD_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
11663 case MVT::v4f16: return fastEmit_ISD_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11664 case MVT::v8f16: return fastEmit_ISD_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11665 case MVT::v2f32: return fastEmit_ISD_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11666 case MVT::v4f32: return fastEmit_ISD_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11667 case MVT::v1f64: return fastEmit_ISD_FMINNUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11668 case MVT::v2f64: return fastEmit_ISD_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11669 default: return Register();
11670 }
11671}
11672
11673// FastEmit functions for ISD::FMINNUM_IEEE.
11674
11675Register fastEmit_ISD_FMINNUM_IEEE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11676 if (RetVT.SimpleTy != MVT::f16)
11677 return Register();
11678 if ((Subtarget->hasFullFP16())) {
11679 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11680 }
11681 return Register();
11682}
11683
11684Register fastEmit_ISD_FMINNUM_IEEE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11685 if (RetVT.SimpleTy != MVT::f32)
11686 return Register();
11687 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11688}
11689
11690Register fastEmit_ISD_FMINNUM_IEEE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11691 if (RetVT.SimpleTy != MVT::f64)
11692 return Register();
11693 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11694}
11695
11696Register fastEmit_ISD_FMINNUM_IEEE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11697 if (RetVT.SimpleTy != MVT::v4f16)
11698 return Register();
11699 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11700 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11701 }
11702 return Register();
11703}
11704
11705Register fastEmit_ISD_FMINNUM_IEEE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11706 if (RetVT.SimpleTy != MVT::v8f16)
11707 return Register();
11708 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11709 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11710 }
11711 return Register();
11712}
11713
11714Register fastEmit_ISD_FMINNUM_IEEE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11715 if (RetVT.SimpleTy != MVT::v2f32)
11716 return Register();
11717 if ((Subtarget->isNeonAvailable())) {
11718 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11719 }
11720 return Register();
11721}
11722
11723Register fastEmit_ISD_FMINNUM_IEEE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11724 if (RetVT.SimpleTy != MVT::v4f32)
11725 return Register();
11726 if ((Subtarget->isNeonAvailable())) {
11727 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11728 }
11729 return Register();
11730}
11731
11732Register fastEmit_ISD_FMINNUM_IEEE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11733 if (RetVT.SimpleTy != MVT::v2f64)
11734 return Register();
11735 if ((Subtarget->isNeonAvailable())) {
11736 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11737 }
11738 return Register();
11739}
11740
11741Register fastEmit_ISD_FMINNUM_IEEE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11742 switch (VT.SimpleTy) {
11743 case MVT::f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_f16_rr(RetVT, Op0, Op1);
11744 case MVT::f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_f32_rr(RetVT, Op0, Op1);
11745 case MVT::f64: return fastEmit_ISD_FMINNUM_IEEE_MVT_f64_rr(RetVT, Op0, Op1);
11746 case MVT::v4f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_v4f16_rr(RetVT, Op0, Op1);
11747 case MVT::v8f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_v8f16_rr(RetVT, Op0, Op1);
11748 case MVT::v2f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_v2f32_rr(RetVT, Op0, Op1);
11749 case MVT::v4f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_v4f32_rr(RetVT, Op0, Op1);
11750 case MVT::v2f64: return fastEmit_ISD_FMINNUM_IEEE_MVT_v2f64_rr(RetVT, Op0, Op1);
11751 default: return Register();
11752 }
11753}
11754
11755// FastEmit functions for ISD::FMUL.
11756
11757Register fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11758 if (RetVT.SimpleTy != MVT::f16)
11759 return Register();
11760 if ((Subtarget->hasFullFP16())) {
11761 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11762 }
11763 return Register();
11764}
11765
11766Register fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11767 if (RetVT.SimpleTy != MVT::f32)
11768 return Register();
11769 if ((Subtarget->hasFPARMv8())) {
11770 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11771 }
11772 return Register();
11773}
11774
11775Register fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11776 if (RetVT.SimpleTy != MVT::f64)
11777 return Register();
11778 if ((Subtarget->hasFPARMv8())) {
11779 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11780 }
11781 return Register();
11782}
11783
11784Register fastEmit_ISD_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11785 if (RetVT.SimpleTy != MVT::v4f16)
11786 return Register();
11787 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11788 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11789 }
11790 return Register();
11791}
11792
11793Register fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11794 if (RetVT.SimpleTy != MVT::v8f16)
11795 return Register();
11796 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11797 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11798 }
11799 return Register();
11800}
11801
11802Register fastEmit_ISD_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11803 if (RetVT.SimpleTy != MVT::v2f32)
11804 return Register();
11805 if ((Subtarget->isNeonAvailable())) {
11806 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11807 }
11808 return Register();
11809}
11810
11811Register fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11812 if (RetVT.SimpleTy != MVT::v4f32)
11813 return Register();
11814 if ((Subtarget->isNeonAvailable())) {
11815 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11816 }
11817 return Register();
11818}
11819
11820Register fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11821 if (RetVT.SimpleTy != MVT::v2f64)
11822 return Register();
11823 if ((Subtarget->isNeonAvailable())) {
11824 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11825 }
11826 return Register();
11827}
11828
11829Register fastEmit_ISD_FMUL_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11830 if (RetVT.SimpleTy != MVT::nxv8f16)
11831 return Register();
11832 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11833 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11834 }
11835 return Register();
11836}
11837
11838Register fastEmit_ISD_FMUL_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11839 if (RetVT.SimpleTy != MVT::nxv8bf16)
11840 return Register();
11841 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
11842 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFMUL_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11843 }
11844 return Register();
11845}
11846
11847Register fastEmit_ISD_FMUL_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11848 if (RetVT.SimpleTy != MVT::nxv4f32)
11849 return Register();
11850 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11851 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11852 }
11853 return Register();
11854}
11855
11856Register fastEmit_ISD_FMUL_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11857 if (RetVT.SimpleTy != MVT::nxv2f64)
11858 return Register();
11859 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11860 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11861 }
11862 return Register();
11863}
11864
11865Register fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11866 switch (VT.SimpleTy) {
11867 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
11868 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
11869 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
11870 case MVT::v4f16: return fastEmit_ISD_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
11871 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
11872 case MVT::v2f32: return fastEmit_ISD_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
11873 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
11874 case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
11875 case MVT::nxv8f16: return fastEmit_ISD_FMUL_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11876 case MVT::nxv8bf16: return fastEmit_ISD_FMUL_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11877 case MVT::nxv4f32: return fastEmit_ISD_FMUL_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11878 case MVT::nxv2f64: return fastEmit_ISD_FMUL_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11879 default: return Register();
11880 }
11881}
11882
11883// FastEmit functions for ISD::FSUB.
11884
11885Register fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11886 if (RetVT.SimpleTy != MVT::f16)
11887 return Register();
11888 if ((Subtarget->hasFullFP16())) {
11889 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11890 }
11891 return Register();
11892}
11893
11894Register fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11895 if (RetVT.SimpleTy != MVT::f32)
11896 return Register();
11897 if ((Subtarget->hasFPARMv8())) {
11898 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11899 }
11900 return Register();
11901}
11902
11903Register fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11904 if (RetVT.SimpleTy != MVT::f64)
11905 return Register();
11906 if ((Subtarget->hasFPARMv8())) {
11907 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11908 }
11909 return Register();
11910}
11911
11912Register fastEmit_ISD_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11913 if (RetVT.SimpleTy != MVT::v4f16)
11914 return Register();
11915 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11916 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11917 }
11918 return Register();
11919}
11920
11921Register fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11922 if (RetVT.SimpleTy != MVT::v8f16)
11923 return Register();
11924 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11925 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11926 }
11927 return Register();
11928}
11929
11930Register fastEmit_ISD_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11931 if (RetVT.SimpleTy != MVT::v2f32)
11932 return Register();
11933 if ((Subtarget->isNeonAvailable())) {
11934 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11935 }
11936 return Register();
11937}
11938
11939Register fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11940 if (RetVT.SimpleTy != MVT::v4f32)
11941 return Register();
11942 if ((Subtarget->isNeonAvailable())) {
11943 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11944 }
11945 return Register();
11946}
11947
11948Register fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11949 if (RetVT.SimpleTy != MVT::v2f64)
11950 return Register();
11951 if ((Subtarget->isNeonAvailable())) {
11952 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11953 }
11954 return Register();
11955}
11956
11957Register fastEmit_ISD_FSUB_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11958 if (RetVT.SimpleTy != MVT::nxv8f16)
11959 return Register();
11960 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11961 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11962 }
11963 return Register();
11964}
11965
11966Register fastEmit_ISD_FSUB_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11967 if (RetVT.SimpleTy != MVT::nxv8bf16)
11968 return Register();
11969 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
11970 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFSUB_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11971 }
11972 return Register();
11973}
11974
11975Register fastEmit_ISD_FSUB_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11976 if (RetVT.SimpleTy != MVT::nxv4f32)
11977 return Register();
11978 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11979 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11980 }
11981 return Register();
11982}
11983
11984Register fastEmit_ISD_FSUB_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11985 if (RetVT.SimpleTy != MVT::nxv2f64)
11986 return Register();
11987 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11988 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11989 }
11990 return Register();
11991}
11992
11993Register fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11994 switch (VT.SimpleTy) {
11995 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
11996 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
11997 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
11998 case MVT::v4f16: return fastEmit_ISD_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
11999 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
12000 case MVT::v2f32: return fastEmit_ISD_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
12001 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
12002 case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
12003 case MVT::nxv8f16: return fastEmit_ISD_FSUB_MVT_nxv8f16_rr(RetVT, Op0, Op1);
12004 case MVT::nxv8bf16: return fastEmit_ISD_FSUB_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
12005 case MVT::nxv4f32: return fastEmit_ISD_FSUB_MVT_nxv4f32_rr(RetVT, Op0, Op1);
12006 case MVT::nxv2f64: return fastEmit_ISD_FSUB_MVT_nxv2f64_rr(RetVT, Op0, Op1);
12007 default: return Register();
12008 }
12009}
12010
12011// FastEmit functions for ISD::GET_ACTIVE_LANE_MASK.
12012
12013Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv2i1_rr(Register Op0, Register Op1) {
12014 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12015 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_D, RC: &AArch64::PPRRegClass, Op0, Op1);
12016 }
12017 return Register();
12018}
12019
12020Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv4i1_rr(Register Op0, Register Op1) {
12021 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12022 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_S, RC: &AArch64::PPRRegClass, Op0, Op1);
12023 }
12024 return Register();
12025}
12026
12027Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv8i1_rr(Register Op0, Register Op1) {
12028 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12029 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_H, RC: &AArch64::PPRRegClass, Op0, Op1);
12030 }
12031 return Register();
12032}
12033
12034Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv16i1_rr(Register Op0, Register Op1) {
12035 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12036 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_B, RC: &AArch64::PPRRegClass, Op0, Op1);
12037 }
12038 return Register();
12039}
12040
12041Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12042switch (RetVT.SimpleTy) {
12043 case MVT::nxv2i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv2i1_rr(Op0, Op1);
12044 case MVT::nxv4i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv4i1_rr(Op0, Op1);
12045 case MVT::nxv8i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv8i1_rr(Op0, Op1);
12046 case MVT::nxv16i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv16i1_rr(Op0, Op1);
12047 default: return Register();
12048}
12049}
12050
12051Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv2i1_rr(Register Op0, Register Op1) {
12052 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12053 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_D, RC: &AArch64::PPRRegClass, Op0, Op1);
12054 }
12055 return Register();
12056}
12057
12058Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv4i1_rr(Register Op0, Register Op1) {
12059 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12060 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_S, RC: &AArch64::PPRRegClass, Op0, Op1);
12061 }
12062 return Register();
12063}
12064
12065Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv8i1_rr(Register Op0, Register Op1) {
12066 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12067 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_H, RC: &AArch64::PPRRegClass, Op0, Op1);
12068 }
12069 return Register();
12070}
12071
12072Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv16i1_rr(Register Op0, Register Op1) {
12073 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12074 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_B, RC: &AArch64::PPRRegClass, Op0, Op1);
12075 }
12076 return Register();
12077}
12078
12079Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12080switch (RetVT.SimpleTy) {
12081 case MVT::nxv2i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv2i1_rr(Op0, Op1);
12082 case MVT::nxv4i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv4i1_rr(Op0, Op1);
12083 case MVT::nxv8i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv8i1_rr(Op0, Op1);
12084 case MVT::nxv16i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv16i1_rr(Op0, Op1);
12085 default: return Register();
12086}
12087}
12088
12089Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12090 switch (VT.SimpleTy) {
12091 case MVT::i32: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_rr(RetVT, Op0, Op1);
12092 case MVT::i64: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_rr(RetVT, Op0, Op1);
12093 default: return Register();
12094 }
12095}
12096
12097// FastEmit functions for ISD::MUL.
12098
12099Register fastEmit_ISD_MUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12100 if (RetVT.SimpleTy != MVT::v8i8)
12101 return Register();
12102 if ((Subtarget->isNeonAvailable())) {
12103 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12104 }
12105 return Register();
12106}
12107
12108Register fastEmit_ISD_MUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12109 if (RetVT.SimpleTy != MVT::v16i8)
12110 return Register();
12111 if ((Subtarget->isNeonAvailable())) {
12112 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12113 }
12114 return Register();
12115}
12116
12117Register fastEmit_ISD_MUL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12118 if (RetVT.SimpleTy != MVT::v4i16)
12119 return Register();
12120 if ((Subtarget->isNeonAvailable())) {
12121 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12122 }
12123 return Register();
12124}
12125
12126Register fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12127 if (RetVT.SimpleTy != MVT::v8i16)
12128 return Register();
12129 if ((Subtarget->isNeonAvailable())) {
12130 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12131 }
12132 return Register();
12133}
12134
12135Register fastEmit_ISD_MUL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12136 if (RetVT.SimpleTy != MVT::v2i32)
12137 return Register();
12138 if ((Subtarget->isNeonAvailable())) {
12139 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12140 }
12141 return Register();
12142}
12143
12144Register fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12145 if (RetVT.SimpleTy != MVT::v4i32)
12146 return Register();
12147 if ((Subtarget->isNeonAvailable())) {
12148 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12149 }
12150 return Register();
12151}
12152
12153Register fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12154 switch (VT.SimpleTy) {
12155 case MVT::v8i8: return fastEmit_ISD_MUL_MVT_v8i8_rr(RetVT, Op0, Op1);
12156 case MVT::v16i8: return fastEmit_ISD_MUL_MVT_v16i8_rr(RetVT, Op0, Op1);
12157 case MVT::v4i16: return fastEmit_ISD_MUL_MVT_v4i16_rr(RetVT, Op0, Op1);
12158 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
12159 case MVT::v2i32: return fastEmit_ISD_MUL_MVT_v2i32_rr(RetVT, Op0, Op1);
12160 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
12161 default: return Register();
12162 }
12163}
12164
12165// FastEmit functions for ISD::MULHS.
12166
12167Register fastEmit_ISD_MULHS_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12168 if (RetVT.SimpleTy != MVT::i64)
12169 return Register();
12170 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULHrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12171}
12172
12173Register fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12174 switch (VT.SimpleTy) {
12175 case MVT::i64: return fastEmit_ISD_MULHS_MVT_i64_rr(RetVT, Op0, Op1);
12176 default: return Register();
12177 }
12178}
12179
12180// FastEmit functions for ISD::MULHU.
12181
12182Register fastEmit_ISD_MULHU_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12183 if (RetVT.SimpleTy != MVT::i64)
12184 return Register();
12185 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULHrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12186}
12187
12188Register fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12189 switch (VT.SimpleTy) {
12190 case MVT::i64: return fastEmit_ISD_MULHU_MVT_i64_rr(RetVT, Op0, Op1);
12191 default: return Register();
12192 }
12193}
12194
12195// FastEmit functions for ISD::OR.
12196
12197Register fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12198 if (RetVT.SimpleTy != MVT::i32)
12199 return Register();
12200 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12201}
12202
12203Register fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12204 if (RetVT.SimpleTy != MVT::i64)
12205 return Register();
12206 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12207}
12208
12209Register fastEmit_ISD_OR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12210 if (RetVT.SimpleTy != MVT::v8i8)
12211 return Register();
12212 if ((Subtarget->isNeonAvailable())) {
12213 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12214 }
12215 return Register();
12216}
12217
12218Register fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12219 if (RetVT.SimpleTy != MVT::v16i8)
12220 return Register();
12221 if ((Subtarget->isNeonAvailable())) {
12222 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12223 }
12224 return Register();
12225}
12226
12227Register fastEmit_ISD_OR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12228 if (RetVT.SimpleTy != MVT::v4i16)
12229 return Register();
12230 if ((Subtarget->isNeonAvailable())) {
12231 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12232 }
12233 return Register();
12234}
12235
12236Register fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12237 if (RetVT.SimpleTy != MVT::v8i16)
12238 return Register();
12239 if ((Subtarget->isNeonAvailable())) {
12240 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12241 }
12242 return Register();
12243}
12244
12245Register fastEmit_ISD_OR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12246 if (RetVT.SimpleTy != MVT::v2i32)
12247 return Register();
12248 if ((Subtarget->isNeonAvailable())) {
12249 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12250 }
12251 return Register();
12252}
12253
12254Register fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12255 if (RetVT.SimpleTy != MVT::v4i32)
12256 return Register();
12257 if ((Subtarget->isNeonAvailable())) {
12258 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12259 }
12260 return Register();
12261}
12262
12263Register fastEmit_ISD_OR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12264 if (RetVT.SimpleTy != MVT::v1i64)
12265 return Register();
12266 if ((Subtarget->isNeonAvailable())) {
12267 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12268 }
12269 return Register();
12270}
12271
12272Register fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12273 if (RetVT.SimpleTy != MVT::v2i64)
12274 return Register();
12275 if ((Subtarget->isNeonAvailable())) {
12276 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12277 }
12278 return Register();
12279}
12280
12281Register fastEmit_ISD_OR_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12282 if (RetVT.SimpleTy != MVT::nxv16i8)
12283 return Register();
12284 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12285 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12286 }
12287 return Register();
12288}
12289
12290Register fastEmit_ISD_OR_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12291 if (RetVT.SimpleTy != MVT::nxv8i16)
12292 return Register();
12293 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12294 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12295 }
12296 return Register();
12297}
12298
12299Register fastEmit_ISD_OR_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12300 if (RetVT.SimpleTy != MVT::nxv4i32)
12301 return Register();
12302 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12303 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12304 }
12305 return Register();
12306}
12307
12308Register fastEmit_ISD_OR_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12309 if (RetVT.SimpleTy != MVT::nxv2i64)
12310 return Register();
12311 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12312 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12313 }
12314 return Register();
12315}
12316
12317Register fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12318 switch (VT.SimpleTy) {
12319 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
12320 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op1);
12321 case MVT::v8i8: return fastEmit_ISD_OR_MVT_v8i8_rr(RetVT, Op0, Op1);
12322 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
12323 case MVT::v4i16: return fastEmit_ISD_OR_MVT_v4i16_rr(RetVT, Op0, Op1);
12324 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
12325 case MVT::v2i32: return fastEmit_ISD_OR_MVT_v2i32_rr(RetVT, Op0, Op1);
12326 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
12327 case MVT::v1i64: return fastEmit_ISD_OR_MVT_v1i64_rr(RetVT, Op0, Op1);
12328 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
12329 case MVT::nxv16i8: return fastEmit_ISD_OR_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12330 case MVT::nxv8i16: return fastEmit_ISD_OR_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12331 case MVT::nxv4i32: return fastEmit_ISD_OR_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12332 case MVT::nxv2i64: return fastEmit_ISD_OR_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12333 default: return Register();
12334 }
12335}
12336
12337// FastEmit functions for ISD::ROTR.
12338
12339Register fastEmit_ISD_ROTR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12340 if (RetVT.SimpleTy != MVT::i64)
12341 return Register();
12342 return fastEmitInst_rr(MachineInstOpcode: AArch64::RORVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12343}
12344
12345Register fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12346 switch (VT.SimpleTy) {
12347 case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_rr(RetVT, Op0, Op1);
12348 default: return Register();
12349 }
12350}
12351
12352// FastEmit functions for ISD::SADDSAT.
12353
12354Register fastEmit_ISD_SADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12355 if (RetVT.SimpleTy != MVT::v8i8)
12356 return Register();
12357 if ((Subtarget->isNeonAvailable())) {
12358 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12359 }
12360 return Register();
12361}
12362
12363Register fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12364 if (RetVT.SimpleTy != MVT::v16i8)
12365 return Register();
12366 if ((Subtarget->isNeonAvailable())) {
12367 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12368 }
12369 return Register();
12370}
12371
12372Register fastEmit_ISD_SADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12373 if (RetVT.SimpleTy != MVT::v4i16)
12374 return Register();
12375 if ((Subtarget->isNeonAvailable())) {
12376 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12377 }
12378 return Register();
12379}
12380
12381Register fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12382 if (RetVT.SimpleTy != MVT::v8i16)
12383 return Register();
12384 if ((Subtarget->isNeonAvailable())) {
12385 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12386 }
12387 return Register();
12388}
12389
12390Register fastEmit_ISD_SADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12391 if (RetVT.SimpleTy != MVT::v2i32)
12392 return Register();
12393 if ((Subtarget->isNeonAvailable())) {
12394 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12395 }
12396 return Register();
12397}
12398
12399Register fastEmit_ISD_SADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12400 if (RetVT.SimpleTy != MVT::v4i32)
12401 return Register();
12402 if ((Subtarget->isNeonAvailable())) {
12403 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12404 }
12405 return Register();
12406}
12407
12408Register fastEmit_ISD_SADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12409 if (RetVT.SimpleTy != MVT::v1i64)
12410 return Register();
12411 if ((Subtarget->isNeonAvailable())) {
12412 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
12413 }
12414 return Register();
12415}
12416
12417Register fastEmit_ISD_SADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12418 if (RetVT.SimpleTy != MVT::v2i64)
12419 return Register();
12420 if ((Subtarget->isNeonAvailable())) {
12421 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12422 }
12423 return Register();
12424}
12425
12426Register fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12427 if (RetVT.SimpleTy != MVT::nxv16i8)
12428 return Register();
12429 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12430 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
12431 }
12432 return Register();
12433}
12434
12435Register fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12436 if (RetVT.SimpleTy != MVT::nxv8i16)
12437 return Register();
12438 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12439 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
12440 }
12441 return Register();
12442}
12443
12444Register fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12445 if (RetVT.SimpleTy != MVT::nxv4i32)
12446 return Register();
12447 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12448 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
12449 }
12450 return Register();
12451}
12452
12453Register fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12454 if (RetVT.SimpleTy != MVT::nxv2i64)
12455 return Register();
12456 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12457 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
12458 }
12459 return Register();
12460}
12461
12462Register fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12463 switch (VT.SimpleTy) {
12464 case MVT::v8i8: return fastEmit_ISD_SADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
12465 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12466 case MVT::v4i16: return fastEmit_ISD_SADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
12467 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12468 case MVT::v2i32: return fastEmit_ISD_SADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
12469 case MVT::v4i32: return fastEmit_ISD_SADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
12470 case MVT::v1i64: return fastEmit_ISD_SADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
12471 case MVT::v2i64: return fastEmit_ISD_SADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
12472 case MVT::nxv16i8: return fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12473 case MVT::nxv8i16: return fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12474 case MVT::nxv4i32: return fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12475 case MVT::nxv2i64: return fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12476 default: return Register();
12477 }
12478}
12479
12480// FastEmit functions for ISD::SDIV.
12481
12482Register fastEmit_ISD_SDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12483 if (RetVT.SimpleTy != MVT::i32)
12484 return Register();
12485 return fastEmitInst_rr(MachineInstOpcode: AArch64::SDIVWr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12486}
12487
12488Register fastEmit_ISD_SDIV_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12489 if (RetVT.SimpleTy != MVT::i64)
12490 return Register();
12491 return fastEmitInst_rr(MachineInstOpcode: AArch64::SDIVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12492}
12493
12494Register fastEmit_ISD_SDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12495 switch (VT.SimpleTy) {
12496 case MVT::i32: return fastEmit_ISD_SDIV_MVT_i32_rr(RetVT, Op0, Op1);
12497 case MVT::i64: return fastEmit_ISD_SDIV_MVT_i64_rr(RetVT, Op0, Op1);
12498 default: return Register();
12499 }
12500}
12501
12502// FastEmit functions for ISD::SHL.
12503
12504Register fastEmit_ISD_SHL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12505 if (RetVT.SimpleTy != MVT::i64)
12506 return Register();
12507 return fastEmitInst_rr(MachineInstOpcode: AArch64::LSLVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12508}
12509
12510Register fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12511 switch (VT.SimpleTy) {
12512 case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_rr(RetVT, Op0, Op1);
12513 default: return Register();
12514 }
12515}
12516
12517// FastEmit functions for ISD::SMAX.
12518
12519Register fastEmit_ISD_SMAX_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12520 if (RetVT.SimpleTy != MVT::i32)
12521 return Register();
12522 if ((Subtarget->hasCSSC())) {
12523 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12524 }
12525 return Register();
12526}
12527
12528Register fastEmit_ISD_SMAX_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12529 if (RetVT.SimpleTy != MVT::i64)
12530 return Register();
12531 if ((Subtarget->hasCSSC())) {
12532 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12533 }
12534 return Register();
12535}
12536
12537Register fastEmit_ISD_SMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12538 if (RetVT.SimpleTy != MVT::v8i8)
12539 return Register();
12540 if ((Subtarget->isNeonAvailable())) {
12541 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12542 }
12543 return Register();
12544}
12545
12546Register fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12547 if (RetVT.SimpleTy != MVT::v16i8)
12548 return Register();
12549 if ((Subtarget->isNeonAvailable())) {
12550 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12551 }
12552 return Register();
12553}
12554
12555Register fastEmit_ISD_SMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12556 if (RetVT.SimpleTy != MVT::v4i16)
12557 return Register();
12558 if ((Subtarget->isNeonAvailable())) {
12559 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12560 }
12561 return Register();
12562}
12563
12564Register fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12565 if (RetVT.SimpleTy != MVT::v8i16)
12566 return Register();
12567 if ((Subtarget->isNeonAvailable())) {
12568 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12569 }
12570 return Register();
12571}
12572
12573Register fastEmit_ISD_SMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12574 if (RetVT.SimpleTy != MVT::v2i32)
12575 return Register();
12576 if ((Subtarget->isNeonAvailable())) {
12577 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12578 }
12579 return Register();
12580}
12581
12582Register fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12583 if (RetVT.SimpleTy != MVT::v4i32)
12584 return Register();
12585 if ((Subtarget->isNeonAvailable())) {
12586 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12587 }
12588 return Register();
12589}
12590
12591Register fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12592 switch (VT.SimpleTy) {
12593 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_rr(RetVT, Op0, Op1);
12594 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_rr(RetVT, Op0, Op1);
12595 case MVT::v8i8: return fastEmit_ISD_SMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
12596 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
12597 case MVT::v4i16: return fastEmit_ISD_SMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
12598 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
12599 case MVT::v2i32: return fastEmit_ISD_SMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
12600 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
12601 default: return Register();
12602 }
12603}
12604
12605// FastEmit functions for ISD::SMIN.
12606
12607Register fastEmit_ISD_SMIN_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12608 if (RetVT.SimpleTy != MVT::i32)
12609 return Register();
12610 if ((Subtarget->hasCSSC())) {
12611 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12612 }
12613 return Register();
12614}
12615
12616Register fastEmit_ISD_SMIN_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12617 if (RetVT.SimpleTy != MVT::i64)
12618 return Register();
12619 if ((Subtarget->hasCSSC())) {
12620 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12621 }
12622 return Register();
12623}
12624
12625Register fastEmit_ISD_SMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12626 if (RetVT.SimpleTy != MVT::v8i8)
12627 return Register();
12628 if ((Subtarget->isNeonAvailable())) {
12629 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12630 }
12631 return Register();
12632}
12633
12634Register fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12635 if (RetVT.SimpleTy != MVT::v16i8)
12636 return Register();
12637 if ((Subtarget->isNeonAvailable())) {
12638 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12639 }
12640 return Register();
12641}
12642
12643Register fastEmit_ISD_SMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12644 if (RetVT.SimpleTy != MVT::v4i16)
12645 return Register();
12646 if ((Subtarget->isNeonAvailable())) {
12647 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12648 }
12649 return Register();
12650}
12651
12652Register fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12653 if (RetVT.SimpleTy != MVT::v8i16)
12654 return Register();
12655 if ((Subtarget->isNeonAvailable())) {
12656 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12657 }
12658 return Register();
12659}
12660
12661Register fastEmit_ISD_SMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12662 if (RetVT.SimpleTy != MVT::v2i32)
12663 return Register();
12664 if ((Subtarget->isNeonAvailable())) {
12665 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12666 }
12667 return Register();
12668}
12669
12670Register fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12671 if (RetVT.SimpleTy != MVT::v4i32)
12672 return Register();
12673 if ((Subtarget->isNeonAvailable())) {
12674 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12675 }
12676 return Register();
12677}
12678
12679Register fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12680 switch (VT.SimpleTy) {
12681 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_rr(RetVT, Op0, Op1);
12682 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_rr(RetVT, Op0, Op1);
12683 case MVT::v8i8: return fastEmit_ISD_SMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
12684 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
12685 case MVT::v4i16: return fastEmit_ISD_SMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
12686 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
12687 case MVT::v2i32: return fastEmit_ISD_SMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
12688 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
12689 default: return Register();
12690 }
12691}
12692
12693// FastEmit functions for ISD::SRA.
12694
12695Register fastEmit_ISD_SRA_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12696 if (RetVT.SimpleTy != MVT::i64)
12697 return Register();
12698 return fastEmitInst_rr(MachineInstOpcode: AArch64::ASRVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12699}
12700
12701Register fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12702 switch (VT.SimpleTy) {
12703 case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_rr(RetVT, Op0, Op1);
12704 default: return Register();
12705 }
12706}
12707
12708// FastEmit functions for ISD::SRL.
12709
12710Register fastEmit_ISD_SRL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12711 if (RetVT.SimpleTy != MVT::i64)
12712 return Register();
12713 return fastEmitInst_rr(MachineInstOpcode: AArch64::LSRVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12714}
12715
12716Register fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12717 switch (VT.SimpleTy) {
12718 case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_rr(RetVT, Op0, Op1);
12719 default: return Register();
12720 }
12721}
12722
12723// FastEmit functions for ISD::SSUBSAT.
12724
12725Register fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12726 if (RetVT.SimpleTy != MVT::v8i8)
12727 return Register();
12728 if ((Subtarget->isNeonAvailable())) {
12729 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12730 }
12731 return Register();
12732}
12733
12734Register fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12735 if (RetVT.SimpleTy != MVT::v16i8)
12736 return Register();
12737 if ((Subtarget->isNeonAvailable())) {
12738 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12739 }
12740 return Register();
12741}
12742
12743Register fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12744 if (RetVT.SimpleTy != MVT::v4i16)
12745 return Register();
12746 if ((Subtarget->isNeonAvailable())) {
12747 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12748 }
12749 return Register();
12750}
12751
12752Register fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12753 if (RetVT.SimpleTy != MVT::v8i16)
12754 return Register();
12755 if ((Subtarget->isNeonAvailable())) {
12756 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12757 }
12758 return Register();
12759}
12760
12761Register fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12762 if (RetVT.SimpleTy != MVT::v2i32)
12763 return Register();
12764 if ((Subtarget->isNeonAvailable())) {
12765 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12766 }
12767 return Register();
12768}
12769
12770Register fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12771 if (RetVT.SimpleTy != MVT::v4i32)
12772 return Register();
12773 if ((Subtarget->isNeonAvailable())) {
12774 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12775 }
12776 return Register();
12777}
12778
12779Register fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12780 if (RetVT.SimpleTy != MVT::v1i64)
12781 return Register();
12782 if ((Subtarget->isNeonAvailable())) {
12783 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
12784 }
12785 return Register();
12786}
12787
12788Register fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12789 if (RetVT.SimpleTy != MVT::v2i64)
12790 return Register();
12791 if ((Subtarget->isNeonAvailable())) {
12792 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12793 }
12794 return Register();
12795}
12796
12797Register fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12798 if (RetVT.SimpleTy != MVT::nxv16i8)
12799 return Register();
12800 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12801 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
12802 }
12803 return Register();
12804}
12805
12806Register fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12807 if (RetVT.SimpleTy != MVT::nxv8i16)
12808 return Register();
12809 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12810 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
12811 }
12812 return Register();
12813}
12814
12815Register fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12816 if (RetVT.SimpleTy != MVT::nxv4i32)
12817 return Register();
12818 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12819 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
12820 }
12821 return Register();
12822}
12823
12824Register fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12825 if (RetVT.SimpleTy != MVT::nxv2i64)
12826 return Register();
12827 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12828 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
12829 }
12830 return Register();
12831}
12832
12833Register fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12834 switch (VT.SimpleTy) {
12835 case MVT::v8i8: return fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
12836 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12837 case MVT::v4i16: return fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
12838 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12839 case MVT::v2i32: return fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
12840 case MVT::v4i32: return fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
12841 case MVT::v1i64: return fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
12842 case MVT::v2i64: return fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
12843 case MVT::nxv16i8: return fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12844 case MVT::nxv8i16: return fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12845 case MVT::nxv4i32: return fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12846 case MVT::nxv2i64: return fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12847 default: return Register();
12848 }
12849}
12850
12851// FastEmit functions for ISD::STRICT_FADD.
12852
12853Register fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12854 if (RetVT.SimpleTy != MVT::f16)
12855 return Register();
12856 if ((Subtarget->hasFullFP16())) {
12857 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12858 }
12859 return Register();
12860}
12861
12862Register fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12863 if (RetVT.SimpleTy != MVT::f32)
12864 return Register();
12865 if ((Subtarget->hasFPARMv8())) {
12866 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12867 }
12868 return Register();
12869}
12870
12871Register fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12872 if (RetVT.SimpleTy != MVT::f64)
12873 return Register();
12874 if ((Subtarget->hasFPARMv8())) {
12875 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12876 }
12877 return Register();
12878}
12879
12880Register fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12881 if (RetVT.SimpleTy != MVT::v4f16)
12882 return Register();
12883 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12884 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12885 }
12886 return Register();
12887}
12888
12889Register fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12890 if (RetVT.SimpleTy != MVT::v8f16)
12891 return Register();
12892 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12893 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12894 }
12895 return Register();
12896}
12897
12898Register fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12899 if (RetVT.SimpleTy != MVT::v2f32)
12900 return Register();
12901 if ((Subtarget->isNeonAvailable())) {
12902 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12903 }
12904 return Register();
12905}
12906
12907Register fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12908 if (RetVT.SimpleTy != MVT::v4f32)
12909 return Register();
12910 if ((Subtarget->isNeonAvailable())) {
12911 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12912 }
12913 return Register();
12914}
12915
12916Register fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12917 if (RetVT.SimpleTy != MVT::v2f64)
12918 return Register();
12919 if ((Subtarget->isNeonAvailable())) {
12920 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12921 }
12922 return Register();
12923}
12924
12925Register fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12926 switch (VT.SimpleTy) {
12927 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1);
12928 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1);
12929 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1);
12930 case MVT::v4f16: return fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
12931 case MVT::v8f16: return fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
12932 case MVT::v2f32: return fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
12933 case MVT::v4f32: return fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
12934 case MVT::v2f64: return fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
12935 default: return Register();
12936 }
12937}
12938
12939// FastEmit functions for ISD::STRICT_FDIV.
12940
12941Register fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12942 if (RetVT.SimpleTy != MVT::f16)
12943 return Register();
12944 if ((Subtarget->hasFullFP16())) {
12945 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12946 }
12947 return Register();
12948}
12949
12950Register fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12951 if (RetVT.SimpleTy != MVT::f32)
12952 return Register();
12953 if ((Subtarget->hasFPARMv8())) {
12954 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12955 }
12956 return Register();
12957}
12958
12959Register fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12960 if (RetVT.SimpleTy != MVT::f64)
12961 return Register();
12962 if ((Subtarget->hasFPARMv8())) {
12963 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12964 }
12965 return Register();
12966}
12967
12968Register fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12969 if (RetVT.SimpleTy != MVT::v4f16)
12970 return Register();
12971 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12972 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12973 }
12974 return Register();
12975}
12976
12977Register fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12978 if (RetVT.SimpleTy != MVT::v8f16)
12979 return Register();
12980 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12981 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12982 }
12983 return Register();
12984}
12985
12986Register fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12987 if (RetVT.SimpleTy != MVT::v2f32)
12988 return Register();
12989 if ((Subtarget->isNeonAvailable())) {
12990 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12991 }
12992 return Register();
12993}
12994
12995Register fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12996 if (RetVT.SimpleTy != MVT::v4f32)
12997 return Register();
12998 if ((Subtarget->isNeonAvailable())) {
12999 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13000 }
13001 return Register();
13002}
13003
13004Register fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13005 if (RetVT.SimpleTy != MVT::v2f64)
13006 return Register();
13007 if ((Subtarget->isNeonAvailable())) {
13008 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13009 }
13010 return Register();
13011}
13012
13013Register fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13014 switch (VT.SimpleTy) {
13015 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
13016 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
13017 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
13018 case MVT::v4f16: return fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1);
13019 case MVT::v8f16: return fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
13020 case MVT::v2f32: return fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1);
13021 case MVT::v4f32: return fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
13022 case MVT::v2f64: return fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
13023 default: return Register();
13024 }
13025}
13026
13027// FastEmit functions for ISD::STRICT_FMAXIMUM.
13028
13029Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13030 if (RetVT.SimpleTy != MVT::f16)
13031 return Register();
13032 if ((Subtarget->hasFullFP16())) {
13033 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13034 }
13035 return Register();
13036}
13037
13038Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13039 if (RetVT.SimpleTy != MVT::f32)
13040 return Register();
13041 if ((Subtarget->hasFPARMv8())) {
13042 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13043 }
13044 return Register();
13045}
13046
13047Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13048 if (RetVT.SimpleTy != MVT::f64)
13049 return Register();
13050 if ((Subtarget->hasFPARMv8())) {
13051 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13052 }
13053 return Register();
13054}
13055
13056Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13057 if (RetVT.SimpleTy != MVT::v4f16)
13058 return Register();
13059 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13060 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13061 }
13062 return Register();
13063}
13064
13065Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13066 if (RetVT.SimpleTy != MVT::v8f16)
13067 return Register();
13068 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13069 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13070 }
13071 return Register();
13072}
13073
13074Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13075 if (RetVT.SimpleTy != MVT::v2f32)
13076 return Register();
13077 if ((Subtarget->isNeonAvailable())) {
13078 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13079 }
13080 return Register();
13081}
13082
13083Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13084 if (RetVT.SimpleTy != MVT::v4f32)
13085 return Register();
13086 if ((Subtarget->isNeonAvailable())) {
13087 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13088 }
13089 return Register();
13090}
13091
13092Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13093 if (RetVT.SimpleTy != MVT::v2f64)
13094 return Register();
13095 if ((Subtarget->isNeonAvailable())) {
13096 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13097 }
13098 return Register();
13099}
13100
13101Register fastEmit_ISD_STRICT_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13102 switch (VT.SimpleTy) {
13103 case MVT::f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1);
13104 case MVT::f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1);
13105 case MVT::f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1);
13106 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13107 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13108 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13109 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13110 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13111 default: return Register();
13112 }
13113}
13114
13115// FastEmit functions for ISD::STRICT_FMAXNUM.
13116
13117Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13118 if (RetVT.SimpleTy != MVT::f16)
13119 return Register();
13120 if ((Subtarget->hasFullFP16())) {
13121 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13122 }
13123 return Register();
13124}
13125
13126Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13127 if (RetVT.SimpleTy != MVT::f32)
13128 return Register();
13129 if ((Subtarget->hasFPARMv8())) {
13130 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13131 }
13132 return Register();
13133}
13134
13135Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13136 if (RetVT.SimpleTy != MVT::f64)
13137 return Register();
13138 if ((Subtarget->hasFPARMv8())) {
13139 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13140 }
13141 return Register();
13142}
13143
13144Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13145 if (RetVT.SimpleTy != MVT::v4f16)
13146 return Register();
13147 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13148 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13149 }
13150 return Register();
13151}
13152
13153Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13154 if (RetVT.SimpleTy != MVT::v8f16)
13155 return Register();
13156 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13157 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13158 }
13159 return Register();
13160}
13161
13162Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13163 if (RetVT.SimpleTy != MVT::v2f32)
13164 return Register();
13165 if ((Subtarget->isNeonAvailable())) {
13166 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13167 }
13168 return Register();
13169}
13170
13171Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13172 if (RetVT.SimpleTy != MVT::v4f32)
13173 return Register();
13174 if ((Subtarget->isNeonAvailable())) {
13175 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13176 }
13177 return Register();
13178}
13179
13180Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13181 if (RetVT.SimpleTy != MVT::v2f64)
13182 return Register();
13183 if ((Subtarget->isNeonAvailable())) {
13184 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13185 }
13186 return Register();
13187}
13188
13189Register fastEmit_ISD_STRICT_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13190 switch (VT.SimpleTy) {
13191 case MVT::f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
13192 case MVT::f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
13193 case MVT::f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
13194 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13195 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13196 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13197 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13198 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13199 default: return Register();
13200 }
13201}
13202
13203// FastEmit functions for ISD::STRICT_FMINIMUM.
13204
13205Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13206 if (RetVT.SimpleTy != MVT::f16)
13207 return Register();
13208 if ((Subtarget->hasFullFP16())) {
13209 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13210 }
13211 return Register();
13212}
13213
13214Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13215 if (RetVT.SimpleTy != MVT::f32)
13216 return Register();
13217 if ((Subtarget->hasFPARMv8())) {
13218 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13219 }
13220 return Register();
13221}
13222
13223Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13224 if (RetVT.SimpleTy != MVT::f64)
13225 return Register();
13226 if ((Subtarget->hasFPARMv8())) {
13227 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13228 }
13229 return Register();
13230}
13231
13232Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13233 if (RetVT.SimpleTy != MVT::v4f16)
13234 return Register();
13235 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13236 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13237 }
13238 return Register();
13239}
13240
13241Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13242 if (RetVT.SimpleTy != MVT::v8f16)
13243 return Register();
13244 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13245 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13246 }
13247 return Register();
13248}
13249
13250Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13251 if (RetVT.SimpleTy != MVT::v2f32)
13252 return Register();
13253 if ((Subtarget->isNeonAvailable())) {
13254 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13255 }
13256 return Register();
13257}
13258
13259Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13260 if (RetVT.SimpleTy != MVT::v4f32)
13261 return Register();
13262 if ((Subtarget->isNeonAvailable())) {
13263 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13264 }
13265 return Register();
13266}
13267
13268Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13269 if (RetVT.SimpleTy != MVT::v2f64)
13270 return Register();
13271 if ((Subtarget->isNeonAvailable())) {
13272 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13273 }
13274 return Register();
13275}
13276
13277Register fastEmit_ISD_STRICT_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13278 switch (VT.SimpleTy) {
13279 case MVT::f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1);
13280 case MVT::f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1);
13281 case MVT::f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1);
13282 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13283 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13284 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13285 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13286 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13287 default: return Register();
13288 }
13289}
13290
13291// FastEmit functions for ISD::STRICT_FMINNUM.
13292
13293Register fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13294 if (RetVT.SimpleTy != MVT::f16)
13295 return Register();
13296 if ((Subtarget->hasFullFP16())) {
13297 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13298 }
13299 return Register();
13300}
13301
13302Register fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13303 if (RetVT.SimpleTy != MVT::f32)
13304 return Register();
13305 if ((Subtarget->hasFPARMv8())) {
13306 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13307 }
13308 return Register();
13309}
13310
13311Register fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13312 if (RetVT.SimpleTy != MVT::f64)
13313 return Register();
13314 if ((Subtarget->hasFPARMv8())) {
13315 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13316 }
13317 return Register();
13318}
13319
13320Register fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13321 if (RetVT.SimpleTy != MVT::v4f16)
13322 return Register();
13323 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13324 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13325 }
13326 return Register();
13327}
13328
13329Register fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13330 if (RetVT.SimpleTy != MVT::v8f16)
13331 return Register();
13332 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13333 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13334 }
13335 return Register();
13336}
13337
13338Register fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13339 if (RetVT.SimpleTy != MVT::v2f32)
13340 return Register();
13341 if ((Subtarget->isNeonAvailable())) {
13342 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13343 }
13344 return Register();
13345}
13346
13347Register fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13348 if (RetVT.SimpleTy != MVT::v4f32)
13349 return Register();
13350 if ((Subtarget->isNeonAvailable())) {
13351 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13352 }
13353 return Register();
13354}
13355
13356Register fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13357 if (RetVT.SimpleTy != MVT::v2f64)
13358 return Register();
13359 if ((Subtarget->isNeonAvailable())) {
13360 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13361 }
13362 return Register();
13363}
13364
13365Register fastEmit_ISD_STRICT_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13366 switch (VT.SimpleTy) {
13367 case MVT::f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
13368 case MVT::f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
13369 case MVT::f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
13370 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13371 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13372 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13373 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13374 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13375 default: return Register();
13376 }
13377}
13378
13379// FastEmit functions for ISD::STRICT_FMUL.
13380
13381Register fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13382 if (RetVT.SimpleTy != MVT::f16)
13383 return Register();
13384 if ((Subtarget->hasFullFP16())) {
13385 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13386 }
13387 return Register();
13388}
13389
13390Register fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13391 if (RetVT.SimpleTy != MVT::f32)
13392 return Register();
13393 if ((Subtarget->hasFPARMv8())) {
13394 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13395 }
13396 return Register();
13397}
13398
13399Register fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13400 if (RetVT.SimpleTy != MVT::f64)
13401 return Register();
13402 if ((Subtarget->hasFPARMv8())) {
13403 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13404 }
13405 return Register();
13406}
13407
13408Register fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13409 if (RetVT.SimpleTy != MVT::v4f16)
13410 return Register();
13411 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13412 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13413 }
13414 return Register();
13415}
13416
13417Register fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13418 if (RetVT.SimpleTy != MVT::v8f16)
13419 return Register();
13420 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13421 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13422 }
13423 return Register();
13424}
13425
13426Register fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13427 if (RetVT.SimpleTy != MVT::v2f32)
13428 return Register();
13429 if ((Subtarget->isNeonAvailable())) {
13430 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13431 }
13432 return Register();
13433}
13434
13435Register fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13436 if (RetVT.SimpleTy != MVT::v4f32)
13437 return Register();
13438 if ((Subtarget->isNeonAvailable())) {
13439 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13440 }
13441 return Register();
13442}
13443
13444Register fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13445 if (RetVT.SimpleTy != MVT::v2f64)
13446 return Register();
13447 if ((Subtarget->isNeonAvailable())) {
13448 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13449 }
13450 return Register();
13451}
13452
13453Register fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13454 switch (VT.SimpleTy) {
13455 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
13456 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
13457 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
13458 case MVT::v4f16: return fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
13459 case MVT::v8f16: return fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
13460 case MVT::v2f32: return fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
13461 case MVT::v4f32: return fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
13462 case MVT::v2f64: return fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
13463 default: return Register();
13464 }
13465}
13466
13467// FastEmit functions for ISD::STRICT_FSUB.
13468
13469Register fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13470 if (RetVT.SimpleTy != MVT::f16)
13471 return Register();
13472 if ((Subtarget->hasFullFP16())) {
13473 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13474 }
13475 return Register();
13476}
13477
13478Register fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13479 if (RetVT.SimpleTy != MVT::f32)
13480 return Register();
13481 if ((Subtarget->hasFPARMv8())) {
13482 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13483 }
13484 return Register();
13485}
13486
13487Register fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13488 if (RetVT.SimpleTy != MVT::f64)
13489 return Register();
13490 if ((Subtarget->hasFPARMv8())) {
13491 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13492 }
13493 return Register();
13494}
13495
13496Register fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13497 if (RetVT.SimpleTy != MVT::v4f16)
13498 return Register();
13499 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13500 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13501 }
13502 return Register();
13503}
13504
13505Register fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13506 if (RetVT.SimpleTy != MVT::v8f16)
13507 return Register();
13508 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13509 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13510 }
13511 return Register();
13512}
13513
13514Register fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13515 if (RetVT.SimpleTy != MVT::v2f32)
13516 return Register();
13517 if ((Subtarget->isNeonAvailable())) {
13518 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13519 }
13520 return Register();
13521}
13522
13523Register fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13524 if (RetVT.SimpleTy != MVT::v4f32)
13525 return Register();
13526 if ((Subtarget->isNeonAvailable())) {
13527 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13528 }
13529 return Register();
13530}
13531
13532Register fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13533 if (RetVT.SimpleTy != MVT::v2f64)
13534 return Register();
13535 if ((Subtarget->isNeonAvailable())) {
13536 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13537 }
13538 return Register();
13539}
13540
13541Register fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13542 switch (VT.SimpleTy) {
13543 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
13544 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
13545 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
13546 case MVT::v4f16: return fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
13547 case MVT::v8f16: return fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
13548 case MVT::v2f32: return fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
13549 case MVT::v4f32: return fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
13550 case MVT::v2f64: return fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
13551 default: return Register();
13552 }
13553}
13554
13555// FastEmit functions for ISD::SUB.
13556
13557Register fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13558 if (RetVT.SimpleTy != MVT::i32)
13559 return Register();
13560 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBSWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13561}
13562
13563Register fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13564 if (RetVT.SimpleTy != MVT::i64)
13565 return Register();
13566 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBSXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13567}
13568
13569Register fastEmit_ISD_SUB_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13570 if (RetVT.SimpleTy != MVT::v8i8)
13571 return Register();
13572 if ((Subtarget->isNeonAvailable())) {
13573 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13574 }
13575 return Register();
13576}
13577
13578Register fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13579 if (RetVT.SimpleTy != MVT::v16i8)
13580 return Register();
13581 if ((Subtarget->isNeonAvailable())) {
13582 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13583 }
13584 return Register();
13585}
13586
13587Register fastEmit_ISD_SUB_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13588 if (RetVT.SimpleTy != MVT::v4i16)
13589 return Register();
13590 if ((Subtarget->isNeonAvailable())) {
13591 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13592 }
13593 return Register();
13594}
13595
13596Register fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13597 if (RetVT.SimpleTy != MVT::v8i16)
13598 return Register();
13599 if ((Subtarget->isNeonAvailable())) {
13600 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13601 }
13602 return Register();
13603}
13604
13605Register fastEmit_ISD_SUB_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13606 if (RetVT.SimpleTy != MVT::v2i32)
13607 return Register();
13608 if ((Subtarget->isNeonAvailable())) {
13609 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13610 }
13611 return Register();
13612}
13613
13614Register fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13615 if (RetVT.SimpleTy != MVT::v4i32)
13616 return Register();
13617 if ((Subtarget->isNeonAvailable())) {
13618 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13619 }
13620 return Register();
13621}
13622
13623Register fastEmit_ISD_SUB_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13624 if (RetVT.SimpleTy != MVT::v1i64)
13625 return Register();
13626 if ((Subtarget->isNeonAvailable())) {
13627 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13628 }
13629 return Register();
13630}
13631
13632Register fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13633 if (RetVT.SimpleTy != MVT::v2i64)
13634 return Register();
13635 if ((Subtarget->isNeonAvailable())) {
13636 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13637 }
13638 return Register();
13639}
13640
13641Register fastEmit_ISD_SUB_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13642 if (RetVT.SimpleTy != MVT::nxv16i8)
13643 return Register();
13644 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13645 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13646 }
13647 return Register();
13648}
13649
13650Register fastEmit_ISD_SUB_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13651 if (RetVT.SimpleTy != MVT::nxv8i16)
13652 return Register();
13653 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13654 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13655 }
13656 return Register();
13657}
13658
13659Register fastEmit_ISD_SUB_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13660 if (RetVT.SimpleTy != MVT::nxv4i32)
13661 return Register();
13662 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13663 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13664 }
13665 return Register();
13666}
13667
13668Register fastEmit_ISD_SUB_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13669 if (RetVT.SimpleTy != MVT::nxv2i64)
13670 return Register();
13671 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13672 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13673 }
13674 return Register();
13675}
13676
13677Register fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13678 switch (VT.SimpleTy) {
13679 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
13680 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op1);
13681 case MVT::v8i8: return fastEmit_ISD_SUB_MVT_v8i8_rr(RetVT, Op0, Op1);
13682 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
13683 case MVT::v4i16: return fastEmit_ISD_SUB_MVT_v4i16_rr(RetVT, Op0, Op1);
13684 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
13685 case MVT::v2i32: return fastEmit_ISD_SUB_MVT_v2i32_rr(RetVT, Op0, Op1);
13686 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
13687 case MVT::v1i64: return fastEmit_ISD_SUB_MVT_v1i64_rr(RetVT, Op0, Op1);
13688 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
13689 case MVT::nxv16i8: return fastEmit_ISD_SUB_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13690 case MVT::nxv8i16: return fastEmit_ISD_SUB_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13691 case MVT::nxv4i32: return fastEmit_ISD_SUB_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13692 case MVT::nxv2i64: return fastEmit_ISD_SUB_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13693 default: return Register();
13694 }
13695}
13696
13697// FastEmit functions for ISD::UADDSAT.
13698
13699Register fastEmit_ISD_UADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13700 if (RetVT.SimpleTy != MVT::v8i8)
13701 return Register();
13702 if ((Subtarget->isNeonAvailable())) {
13703 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13704 }
13705 return Register();
13706}
13707
13708Register fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13709 if (RetVT.SimpleTy != MVT::v16i8)
13710 return Register();
13711 if ((Subtarget->isNeonAvailable())) {
13712 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13713 }
13714 return Register();
13715}
13716
13717Register fastEmit_ISD_UADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13718 if (RetVT.SimpleTy != MVT::v4i16)
13719 return Register();
13720 if ((Subtarget->isNeonAvailable())) {
13721 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13722 }
13723 return Register();
13724}
13725
13726Register fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13727 if (RetVT.SimpleTy != MVT::v8i16)
13728 return Register();
13729 if ((Subtarget->isNeonAvailable())) {
13730 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13731 }
13732 return Register();
13733}
13734
13735Register fastEmit_ISD_UADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13736 if (RetVT.SimpleTy != MVT::v2i32)
13737 return Register();
13738 if ((Subtarget->isNeonAvailable())) {
13739 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13740 }
13741 return Register();
13742}
13743
13744Register fastEmit_ISD_UADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13745 if (RetVT.SimpleTy != MVT::v4i32)
13746 return Register();
13747 if ((Subtarget->isNeonAvailable())) {
13748 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13749 }
13750 return Register();
13751}
13752
13753Register fastEmit_ISD_UADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13754 if (RetVT.SimpleTy != MVT::v1i64)
13755 return Register();
13756 if ((Subtarget->isNeonAvailable())) {
13757 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13758 }
13759 return Register();
13760}
13761
13762Register fastEmit_ISD_UADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13763 if (RetVT.SimpleTy != MVT::v2i64)
13764 return Register();
13765 if ((Subtarget->isNeonAvailable())) {
13766 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13767 }
13768 return Register();
13769}
13770
13771Register fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13772 if (RetVT.SimpleTy != MVT::nxv16i8)
13773 return Register();
13774 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13775 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13776 }
13777 return Register();
13778}
13779
13780Register fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13781 if (RetVT.SimpleTy != MVT::nxv8i16)
13782 return Register();
13783 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13784 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13785 }
13786 return Register();
13787}
13788
13789Register fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13790 if (RetVT.SimpleTy != MVT::nxv4i32)
13791 return Register();
13792 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13793 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13794 }
13795 return Register();
13796}
13797
13798Register fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13799 if (RetVT.SimpleTy != MVT::nxv2i64)
13800 return Register();
13801 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13802 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13803 }
13804 return Register();
13805}
13806
13807Register fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13808 switch (VT.SimpleTy) {
13809 case MVT::v8i8: return fastEmit_ISD_UADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
13810 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13811 case MVT::v4i16: return fastEmit_ISD_UADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
13812 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13813 case MVT::v2i32: return fastEmit_ISD_UADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
13814 case MVT::v4i32: return fastEmit_ISD_UADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
13815 case MVT::v1i64: return fastEmit_ISD_UADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
13816 case MVT::v2i64: return fastEmit_ISD_UADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
13817 case MVT::nxv16i8: return fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13818 case MVT::nxv8i16: return fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13819 case MVT::nxv4i32: return fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13820 case MVT::nxv2i64: return fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13821 default: return Register();
13822 }
13823}
13824
13825// FastEmit functions for ISD::UDIV.
13826
13827Register fastEmit_ISD_UDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13828 if (RetVT.SimpleTy != MVT::i32)
13829 return Register();
13830 return fastEmitInst_rr(MachineInstOpcode: AArch64::UDIVWr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13831}
13832
13833Register fastEmit_ISD_UDIV_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13834 if (RetVT.SimpleTy != MVT::i64)
13835 return Register();
13836 return fastEmitInst_rr(MachineInstOpcode: AArch64::UDIVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13837}
13838
13839Register fastEmit_ISD_UDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13840 switch (VT.SimpleTy) {
13841 case MVT::i32: return fastEmit_ISD_UDIV_MVT_i32_rr(RetVT, Op0, Op1);
13842 case MVT::i64: return fastEmit_ISD_UDIV_MVT_i64_rr(RetVT, Op0, Op1);
13843 default: return Register();
13844 }
13845}
13846
13847// FastEmit functions for ISD::UMAX.
13848
13849Register fastEmit_ISD_UMAX_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13850 if (RetVT.SimpleTy != MVT::i32)
13851 return Register();
13852 if ((Subtarget->hasCSSC())) {
13853 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13854 }
13855 return Register();
13856}
13857
13858Register fastEmit_ISD_UMAX_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13859 if (RetVT.SimpleTy != MVT::i64)
13860 return Register();
13861 if ((Subtarget->hasCSSC())) {
13862 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13863 }
13864 return Register();
13865}
13866
13867Register fastEmit_ISD_UMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13868 if (RetVT.SimpleTy != MVT::v8i8)
13869 return Register();
13870 if ((Subtarget->isNeonAvailable())) {
13871 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13872 }
13873 return Register();
13874}
13875
13876Register fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13877 if (RetVT.SimpleTy != MVT::v16i8)
13878 return Register();
13879 if ((Subtarget->isNeonAvailable())) {
13880 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13881 }
13882 return Register();
13883}
13884
13885Register fastEmit_ISD_UMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13886 if (RetVT.SimpleTy != MVT::v4i16)
13887 return Register();
13888 if ((Subtarget->isNeonAvailable())) {
13889 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13890 }
13891 return Register();
13892}
13893
13894Register fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13895 if (RetVT.SimpleTy != MVT::v8i16)
13896 return Register();
13897 if ((Subtarget->isNeonAvailable())) {
13898 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13899 }
13900 return Register();
13901}
13902
13903Register fastEmit_ISD_UMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13904 if (RetVT.SimpleTy != MVT::v2i32)
13905 return Register();
13906 if ((Subtarget->isNeonAvailable())) {
13907 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13908 }
13909 return Register();
13910}
13911
13912Register fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13913 if (RetVT.SimpleTy != MVT::v4i32)
13914 return Register();
13915 if ((Subtarget->isNeonAvailable())) {
13916 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13917 }
13918 return Register();
13919}
13920
13921Register fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13922 switch (VT.SimpleTy) {
13923 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_rr(RetVT, Op0, Op1);
13924 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_rr(RetVT, Op0, Op1);
13925 case MVT::v8i8: return fastEmit_ISD_UMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
13926 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
13927 case MVT::v4i16: return fastEmit_ISD_UMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
13928 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
13929 case MVT::v2i32: return fastEmit_ISD_UMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
13930 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
13931 default: return Register();
13932 }
13933}
13934
13935// FastEmit functions for ISD::UMIN.
13936
13937Register fastEmit_ISD_UMIN_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13938 if (RetVT.SimpleTy != MVT::i32)
13939 return Register();
13940 if ((Subtarget->hasCSSC())) {
13941 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13942 }
13943 return Register();
13944}
13945
13946Register fastEmit_ISD_UMIN_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13947 if (RetVT.SimpleTy != MVT::i64)
13948 return Register();
13949 if ((Subtarget->hasCSSC())) {
13950 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13951 }
13952 return Register();
13953}
13954
13955Register fastEmit_ISD_UMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13956 if (RetVT.SimpleTy != MVT::v8i8)
13957 return Register();
13958 if ((Subtarget->isNeonAvailable())) {
13959 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13960 }
13961 return Register();
13962}
13963
13964Register fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13965 if (RetVT.SimpleTy != MVT::v16i8)
13966 return Register();
13967 if ((Subtarget->isNeonAvailable())) {
13968 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13969 }
13970 return Register();
13971}
13972
13973Register fastEmit_ISD_UMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13974 if (RetVT.SimpleTy != MVT::v4i16)
13975 return Register();
13976 if ((Subtarget->isNeonAvailable())) {
13977 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13978 }
13979 return Register();
13980}
13981
13982Register fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13983 if (RetVT.SimpleTy != MVT::v8i16)
13984 return Register();
13985 if ((Subtarget->isNeonAvailable())) {
13986 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13987 }
13988 return Register();
13989}
13990
13991Register fastEmit_ISD_UMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13992 if (RetVT.SimpleTy != MVT::v2i32)
13993 return Register();
13994 if ((Subtarget->isNeonAvailable())) {
13995 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13996 }
13997 return Register();
13998}
13999
14000Register fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14001 if (RetVT.SimpleTy != MVT::v4i32)
14002 return Register();
14003 if ((Subtarget->isNeonAvailable())) {
14004 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
14005 }
14006 return Register();
14007}
14008
14009Register fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14010 switch (VT.SimpleTy) {
14011 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_rr(RetVT, Op0, Op1);
14012 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_rr(RetVT, Op0, Op1);
14013 case MVT::v8i8: return fastEmit_ISD_UMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
14014 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
14015 case MVT::v4i16: return fastEmit_ISD_UMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
14016 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
14017 case MVT::v2i32: return fastEmit_ISD_UMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
14018 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
14019 default: return Register();
14020 }
14021}
14022
14023// FastEmit functions for ISD::USUBSAT.
14024
14025Register fastEmit_ISD_USUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
14026 if (RetVT.SimpleTy != MVT::v8i8)
14027 return Register();
14028 if ((Subtarget->isNeonAvailable())) {
14029 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14030 }
14031 return Register();
14032}
14033
14034Register fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14035 if (RetVT.SimpleTy != MVT::v16i8)
14036 return Register();
14037 if ((Subtarget->isNeonAvailable())) {
14038 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14039 }
14040 return Register();
14041}
14042
14043Register fastEmit_ISD_USUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
14044 if (RetVT.SimpleTy != MVT::v4i16)
14045 return Register();
14046 if ((Subtarget->isNeonAvailable())) {
14047 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
14048 }
14049 return Register();
14050}
14051
14052Register fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14053 if (RetVT.SimpleTy != MVT::v8i16)
14054 return Register();
14055 if ((Subtarget->isNeonAvailable())) {
14056 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
14057 }
14058 return Register();
14059}
14060
14061Register fastEmit_ISD_USUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
14062 if (RetVT.SimpleTy != MVT::v2i32)
14063 return Register();
14064 if ((Subtarget->isNeonAvailable())) {
14065 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
14066 }
14067 return Register();
14068}
14069
14070Register fastEmit_ISD_USUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14071 if (RetVT.SimpleTy != MVT::v4i32)
14072 return Register();
14073 if ((Subtarget->isNeonAvailable())) {
14074 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
14075 }
14076 return Register();
14077}
14078
14079Register fastEmit_ISD_USUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
14080 if (RetVT.SimpleTy != MVT::v1i64)
14081 return Register();
14082 if ((Subtarget->isNeonAvailable())) {
14083 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
14084 }
14085 return Register();
14086}
14087
14088Register fastEmit_ISD_USUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14089 if (RetVT.SimpleTy != MVT::v2i64)
14090 return Register();
14091 if ((Subtarget->isNeonAvailable())) {
14092 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
14093 }
14094 return Register();
14095}
14096
14097Register fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14098 if (RetVT.SimpleTy != MVT::nxv16i8)
14099 return Register();
14100 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14101 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
14102 }
14103 return Register();
14104}
14105
14106Register fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14107 if (RetVT.SimpleTy != MVT::nxv8i16)
14108 return Register();
14109 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14110 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
14111 }
14112 return Register();
14113}
14114
14115Register fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14116 if (RetVT.SimpleTy != MVT::nxv4i32)
14117 return Register();
14118 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14119 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
14120 }
14121 return Register();
14122}
14123
14124Register fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14125 if (RetVT.SimpleTy != MVT::nxv2i64)
14126 return Register();
14127 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14128 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
14129 }
14130 return Register();
14131}
14132
14133Register fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14134 switch (VT.SimpleTy) {
14135 case MVT::v8i8: return fastEmit_ISD_USUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
14136 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
14137 case MVT::v4i16: return fastEmit_ISD_USUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
14138 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
14139 case MVT::v2i32: return fastEmit_ISD_USUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
14140 case MVT::v4i32: return fastEmit_ISD_USUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
14141 case MVT::v1i64: return fastEmit_ISD_USUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
14142 case MVT::v2i64: return fastEmit_ISD_USUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
14143 case MVT::nxv16i8: return fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
14144 case MVT::nxv8i16: return fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
14145 case MVT::nxv4i32: return fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
14146 case MVT::nxv2i64: return fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
14147 default: return Register();
14148 }
14149}
14150
14151// FastEmit functions for ISD::XOR.
14152
14153Register fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
14154 if (RetVT.SimpleTy != MVT::i32)
14155 return Register();
14156 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
14157}
14158
14159Register fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
14160 if (RetVT.SimpleTy != MVT::i64)
14161 return Register();
14162 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
14163}
14164
14165Register fastEmit_ISD_XOR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
14166 if (RetVT.SimpleTy != MVT::v8i8)
14167 return Register();
14168 if ((Subtarget->isNeonAvailable())) {
14169 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14170 }
14171 return Register();
14172}
14173
14174Register fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14175 if (RetVT.SimpleTy != MVT::v16i8)
14176 return Register();
14177 if ((Subtarget->isNeonAvailable())) {
14178 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14179 }
14180 return Register();
14181}
14182
14183Register fastEmit_ISD_XOR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
14184 if (RetVT.SimpleTy != MVT::v4i16)
14185 return Register();
14186 if ((Subtarget->isNeonAvailable())) {
14187 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14188 }
14189 return Register();
14190}
14191
14192Register fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14193 if (RetVT.SimpleTy != MVT::v8i16)
14194 return Register();
14195 if ((Subtarget->isNeonAvailable())) {
14196 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14197 }
14198 return Register();
14199}
14200
14201Register fastEmit_ISD_XOR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
14202 if (RetVT.SimpleTy != MVT::v2i32)
14203 return Register();
14204 if ((Subtarget->isNeonAvailable())) {
14205 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14206 }
14207 return Register();
14208}
14209
14210Register fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14211 if (RetVT.SimpleTy != MVT::v4i32)
14212 return Register();
14213 if ((Subtarget->isNeonAvailable())) {
14214 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14215 }
14216 return Register();
14217}
14218
14219Register fastEmit_ISD_XOR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
14220 if (RetVT.SimpleTy != MVT::v1i64)
14221 return Register();
14222 if ((Subtarget->isNeonAvailable())) {
14223 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14224 }
14225 return Register();
14226}
14227
14228Register fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14229 if (RetVT.SimpleTy != MVT::v2i64)
14230 return Register();
14231 if ((Subtarget->isNeonAvailable())) {
14232 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14233 }
14234 return Register();
14235}
14236
14237Register fastEmit_ISD_XOR_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14238 if (RetVT.SimpleTy != MVT::nxv16i8)
14239 return Register();
14240 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14241 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14242 }
14243 return Register();
14244}
14245
14246Register fastEmit_ISD_XOR_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14247 if (RetVT.SimpleTy != MVT::nxv8i16)
14248 return Register();
14249 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14250 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14251 }
14252 return Register();
14253}
14254
14255Register fastEmit_ISD_XOR_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14256 if (RetVT.SimpleTy != MVT::nxv4i32)
14257 return Register();
14258 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14259 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14260 }
14261 return Register();
14262}
14263
14264Register fastEmit_ISD_XOR_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14265 if (RetVT.SimpleTy != MVT::nxv2i64)
14266 return Register();
14267 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14268 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14269 }
14270 return Register();
14271}
14272
14273Register fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14274 switch (VT.SimpleTy) {
14275 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
14276 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op1);
14277 case MVT::v8i8: return fastEmit_ISD_XOR_MVT_v8i8_rr(RetVT, Op0, Op1);
14278 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
14279 case MVT::v4i16: return fastEmit_ISD_XOR_MVT_v4i16_rr(RetVT, Op0, Op1);
14280 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
14281 case MVT::v2i32: return fastEmit_ISD_XOR_MVT_v2i32_rr(RetVT, Op0, Op1);
14282 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
14283 case MVT::v1i64: return fastEmit_ISD_XOR_MVT_v1i64_rr(RetVT, Op0, Op1);
14284 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
14285 case MVT::nxv16i8: return fastEmit_ISD_XOR_MVT_nxv16i8_rr(RetVT, Op0, Op1);
14286 case MVT::nxv8i16: return fastEmit_ISD_XOR_MVT_nxv8i16_rr(RetVT, Op0, Op1);
14287 case MVT::nxv4i32: return fastEmit_ISD_XOR_MVT_nxv4i32_rr(RetVT, Op0, Op1);
14288 case MVT::nxv2i64: return fastEmit_ISD_XOR_MVT_nxv2i64_rr(RetVT, Op0, Op1);
14289 default: return Register();
14290 }
14291}
14292
14293// Top-level FastEmit function.
14294
14295Register fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, Register Op1) override {
14296 switch (Opcode) {
14297 case AArch64ISD::ADDP: return fastEmit_AArch64ISD_ADDP_rr(VT, RetVT, Op0, Op1);
14298 case AArch64ISD::BIC: return fastEmit_AArch64ISD_BIC_rr(VT, RetVT, Op0, Op1);
14299 case AArch64ISD::FCMEQ: return fastEmit_AArch64ISD_FCMEQ_rr(VT, RetVT, Op0, Op1);
14300 case AArch64ISD::FCMGE: return fastEmit_AArch64ISD_FCMGE_rr(VT, RetVT, Op0, Op1);
14301 case AArch64ISD::FCMGT: return fastEmit_AArch64ISD_FCMGT_rr(VT, RetVT, Op0, Op1);
14302 case AArch64ISD::FCMP: return fastEmit_AArch64ISD_FCMP_rr(VT, RetVT, Op0, Op1);
14303 case AArch64ISD::FRECPS: return fastEmit_AArch64ISD_FRECPS_rr(VT, RetVT, Op0, Op1);
14304 case AArch64ISD::FRSQRTS: return fastEmit_AArch64ISD_FRSQRTS_rr(VT, RetVT, Op0, Op1);
14305 case AArch64ISD::INIT_TPIDR2OBJ: return fastEmit_AArch64ISD_INIT_TPIDR2OBJ_rr(VT, RetVT, Op0, Op1);
14306 case AArch64ISD::PMULL: return fastEmit_AArch64ISD_PMULL_rr(VT, RetVT, Op0, Op1);
14307 case AArch64ISD::PTEST: return fastEmit_AArch64ISD_PTEST_rr(VT, RetVT, Op0, Op1);
14308 case AArch64ISD::PTEST_ANY: return fastEmit_AArch64ISD_PTEST_ANY_rr(VT, RetVT, Op0, Op1);
14309 case AArch64ISD::PTEST_FIRST: return fastEmit_AArch64ISD_PTEST_FIRST_rr(VT, RetVT, Op0, Op1);
14310 case AArch64ISD::SMULL: return fastEmit_AArch64ISD_SMULL_rr(VT, RetVT, Op0, Op1);
14311 case AArch64ISD::SQADD: return fastEmit_AArch64ISD_SQADD_rr(VT, RetVT, Op0, Op1);
14312 case AArch64ISD::SQDMULH: return fastEmit_AArch64ISD_SQDMULH_rr(VT, RetVT, Op0, Op1);
14313 case AArch64ISD::SQDMULL: return fastEmit_AArch64ISD_SQDMULL_rr(VT, RetVT, Op0, Op1);
14314 case AArch64ISD::SQRDMULH: return fastEmit_AArch64ISD_SQRDMULH_rr(VT, RetVT, Op0, Op1);
14315 case AArch64ISD::SQRSHL: return fastEmit_AArch64ISD_SQRSHL_rr(VT, RetVT, Op0, Op1);
14316 case AArch64ISD::SQSHL: return fastEmit_AArch64ISD_SQSHL_rr(VT, RetVT, Op0, Op1);
14317 case AArch64ISD::SQSUB: return fastEmit_AArch64ISD_SQSUB_rr(VT, RetVT, Op0, Op1);
14318 case AArch64ISD::STRICT_FCMP: return fastEmit_AArch64ISD_STRICT_FCMP_rr(VT, RetVT, Op0, Op1);
14319 case AArch64ISD::STRICT_FCMPE: return fastEmit_AArch64ISD_STRICT_FCMPE_rr(VT, RetVT, Op0, Op1);
14320 case AArch64ISD::SUQADD: return fastEmit_AArch64ISD_SUQADD_rr(VT, RetVT, Op0, Op1);
14321 case AArch64ISD::TBL: return fastEmit_AArch64ISD_TBL_rr(VT, RetVT, Op0, Op1);
14322 case AArch64ISD::TRN1: return fastEmit_AArch64ISD_TRN1_rr(VT, RetVT, Op0, Op1);
14323 case AArch64ISD::TRN2: return fastEmit_AArch64ISD_TRN2_rr(VT, RetVT, Op0, Op1);
14324 case AArch64ISD::UMULL: return fastEmit_AArch64ISD_UMULL_rr(VT, RetVT, Op0, Op1);
14325 case AArch64ISD::UQADD: return fastEmit_AArch64ISD_UQADD_rr(VT, RetVT, Op0, Op1);
14326 case AArch64ISD::UQRSHL: return fastEmit_AArch64ISD_UQRSHL_rr(VT, RetVT, Op0, Op1);
14327 case AArch64ISD::UQSHL: return fastEmit_AArch64ISD_UQSHL_rr(VT, RetVT, Op0, Op1);
14328 case AArch64ISD::UQSUB: return fastEmit_AArch64ISD_UQSUB_rr(VT, RetVT, Op0, Op1);
14329 case AArch64ISD::USQADD: return fastEmit_AArch64ISD_USQADD_rr(VT, RetVT, Op0, Op1);
14330 case AArch64ISD::UZP1: return fastEmit_AArch64ISD_UZP1_rr(VT, RetVT, Op0, Op1);
14331 case AArch64ISD::UZP2: return fastEmit_AArch64ISD_UZP2_rr(VT, RetVT, Op0, Op1);
14332 case AArch64ISD::ZIP1: return fastEmit_AArch64ISD_ZIP1_rr(VT, RetVT, Op0, Op1);
14333 case AArch64ISD::ZIP2: return fastEmit_AArch64ISD_ZIP2_rr(VT, RetVT, Op0, Op1);
14334 case ISD::ABDS: return fastEmit_ISD_ABDS_rr(VT, RetVT, Op0, Op1);
14335 case ISD::ABDU: return fastEmit_ISD_ABDU_rr(VT, RetVT, Op0, Op1);
14336 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
14337 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
14338 case ISD::AVGCEILS: return fastEmit_ISD_AVGCEILS_rr(VT, RetVT, Op0, Op1);
14339 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
14340 case ISD::AVGFLOORS: return fastEmit_ISD_AVGFLOORS_rr(VT, RetVT, Op0, Op1);
14341 case ISD::AVGFLOORU: return fastEmit_ISD_AVGFLOORU_rr(VT, RetVT, Op0, Op1);
14342 case ISD::CLMUL: return fastEmit_ISD_CLMUL_rr(VT, RetVT, Op0, Op1);
14343 case ISD::CONCAT_VECTORS: return fastEmit_ISD_CONCAT_VECTORS_rr(VT, RetVT, Op0, Op1);
14344 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
14345 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
14346 case ISD::FMAXIMUM: return fastEmit_ISD_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
14347 case ISD::FMAXNUM: return fastEmit_ISD_FMAXNUM_rr(VT, RetVT, Op0, Op1);
14348 case ISD::FMAXNUM_IEEE: return fastEmit_ISD_FMAXNUM_IEEE_rr(VT, RetVT, Op0, Op1);
14349 case ISD::FMINIMUM: return fastEmit_ISD_FMINIMUM_rr(VT, RetVT, Op0, Op1);
14350 case ISD::FMINNUM: return fastEmit_ISD_FMINNUM_rr(VT, RetVT, Op0, Op1);
14351 case ISD::FMINNUM_IEEE: return fastEmit_ISD_FMINNUM_IEEE_rr(VT, RetVT, Op0, Op1);
14352 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
14353 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
14354 case ISD::GET_ACTIVE_LANE_MASK: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_rr(VT, RetVT, Op0, Op1);
14355 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
14356 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
14357 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
14358 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
14359 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
14360 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
14361 case ISD::SDIV: return fastEmit_ISD_SDIV_rr(VT, RetVT, Op0, Op1);
14362 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
14363 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
14364 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
14365 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
14366 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
14367 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
14368 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1);
14369 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1);
14370 case ISD::STRICT_FMAXIMUM: return fastEmit_ISD_STRICT_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
14371 case ISD::STRICT_FMAXNUM: return fastEmit_ISD_STRICT_FMAXNUM_rr(VT, RetVT, Op0, Op1);
14372 case ISD::STRICT_FMINIMUM: return fastEmit_ISD_STRICT_FMINIMUM_rr(VT, RetVT, Op0, Op1);
14373 case ISD::STRICT_FMINNUM: return fastEmit_ISD_STRICT_FMINNUM_rr(VT, RetVT, Op0, Op1);
14374 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1);
14375 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1);
14376 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
14377 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
14378 case ISD::UDIV: return fastEmit_ISD_UDIV_rr(VT, RetVT, Op0, Op1);
14379 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
14380 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
14381 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
14382 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
14383 default: return Register();
14384 }
14385}
14386
14387// FastEmit functions for AArch64ISD::DUPLANE64.
14388
14389Register fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14390 if (RetVT.SimpleTy != MVT::v2i64)
14391 return Register();
14392 if ((Subtarget->isNeonAvailable())) {
14393 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i64lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14394 }
14395 return Register();
14396}
14397
14398Register fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14399 if (RetVT.SimpleTy != MVT::v2f64)
14400 return Register();
14401 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i64lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14402}
14403
14404Register fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14405 switch (VT.SimpleTy) {
14406 case MVT::v2i64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14407 case MVT::v2f64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14408 default: return Register();
14409 }
14410}
14411
14412// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14413
14414Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14415 if (RetVT.SimpleTy != MVT::i64)
14416 return Register();
14417 if ((Subtarget->isNeonAvailable())) {
14418 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi64, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14419 }
14420 return Register();
14421}
14422
14423Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14424 if (RetVT.SimpleTy != MVT::f64)
14425 return Register();
14426 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi64, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14427}
14428
14429Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14430 switch (VT.SimpleTy) {
14431 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14432 case MVT::v2f64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14433 default: return Register();
14434 }
14435}
14436
14437// Top-level FastEmit function.
14438
14439Register fastEmit_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14440 switch (Opcode) {
14441 case AArch64ISD::DUPLANE64: return fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1);
14442 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1);
14443 default: return Register();
14444 }
14445}
14446
14447// FastEmit functions for AArch64ISD::DUPLANE32.
14448
14449Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14450 if ((Subtarget->isNeonAvailable())) {
14451 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i32lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14452 }
14453 return Register();
14454}
14455
14456Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14457 if ((Subtarget->isNeonAvailable())) {
14458 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i32lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14459 }
14460 return Register();
14461}
14462
14463Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14464switch (RetVT.SimpleTy) {
14465 case MVT::v2i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Op0, imm1);
14466 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Op0, imm1);
14467 default: return Register();
14468}
14469}
14470
14471Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14472 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i32lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14473}
14474
14475Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14476 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i32lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14477}
14478
14479Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14480switch (RetVT.SimpleTy) {
14481 case MVT::v2f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Op0, imm1);
14482 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Op0, imm1);
14483 default: return Register();
14484}
14485}
14486
14487Register fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14488 switch (VT.SimpleTy) {
14489 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14490 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14491 default: return Register();
14492 }
14493}
14494
14495// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14496
14497Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14498 if (RetVT.SimpleTy != MVT::i32)
14499 return Register();
14500 if ((Subtarget->isNeonAvailable())) {
14501 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi32, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14502 }
14503 return Register();
14504}
14505
14506Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14507 if (RetVT.SimpleTy != MVT::f32)
14508 return Register();
14509 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi32, RC: &AArch64::FPR32RegClass, Op0, Imm: imm1);
14510}
14511
14512Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14513 switch (VT.SimpleTy) {
14514 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14515 case MVT::v4f32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14516 default: return Register();
14517 }
14518}
14519
14520// Top-level FastEmit function.
14521
14522Register fastEmit_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14523 switch (Opcode) {
14524 case AArch64ISD::DUPLANE32: return fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1);
14525 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1);
14526 default: return Register();
14527 }
14528}
14529
14530// FastEmit functions for AArch64ISD::DUPLANE16.
14531
14532Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14533 if ((Subtarget->isNeonAvailable())) {
14534 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14535 }
14536 return Register();
14537}
14538
14539Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14540 if ((Subtarget->isNeonAvailable())) {
14541 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14542 }
14543 return Register();
14544}
14545
14546Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14547switch (RetVT.SimpleTy) {
14548 case MVT::v4i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Op0, imm1);
14549 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Op0, imm1);
14550 default: return Register();
14551}
14552}
14553
14554Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14555 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14556}
14557
14558Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14559 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14560}
14561
14562Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14563switch (RetVT.SimpleTy) {
14564 case MVT::v4f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Op0, imm1);
14565 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Op0, imm1);
14566 default: return Register();
14567}
14568}
14569
14570Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14571 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14572}
14573
14574Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14575 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14576}
14577
14578Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14579switch (RetVT.SimpleTy) {
14580 case MVT::v4bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(Op0, imm1);
14581 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(Op0, imm1);
14582 default: return Register();
14583}
14584}
14585
14586Register fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14587 switch (VT.SimpleTy) {
14588 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14589 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14590 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14591 default: return Register();
14592 }
14593}
14594
14595// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14596
14597Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14598 if (RetVT.SimpleTy != MVT::i32)
14599 return Register();
14600 if ((Subtarget->isNeonAvailable())) {
14601 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi16, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14602 }
14603 return Register();
14604}
14605
14606Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14607 if (RetVT.SimpleTy != MVT::f16)
14608 return Register();
14609 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi16, RC: &AArch64::FPR16RegClass, Op0, Imm: imm1);
14610}
14611
14612Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14613 if (RetVT.SimpleTy != MVT::bf16)
14614 return Register();
14615 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi16, RC: &AArch64::FPR16RegClass, Op0, Imm: imm1);
14616}
14617
14618Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14619 switch (VT.SimpleTy) {
14620 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14621 case MVT::v8f16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14622 case MVT::v8bf16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14623 default: return Register();
14624 }
14625}
14626
14627// Top-level FastEmit function.
14628
14629Register fastEmit_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14630 switch (Opcode) {
14631 case AArch64ISD::DUPLANE16: return fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1);
14632 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1);
14633 default: return Register();
14634 }
14635}
14636
14637// FastEmit functions for AArch64ISD::DUPLANE8.
14638
14639Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Register Op0, uint64_t imm1) {
14640 if ((Subtarget->isNeonAvailable())) {
14641 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i8lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14642 }
14643 return Register();
14644}
14645
14646Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Register Op0, uint64_t imm1) {
14647 if ((Subtarget->isNeonAvailable())) {
14648 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv16i8lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14649 }
14650 return Register();
14651}
14652
14653Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, Register Op0, uint64_t imm1) {
14654switch (RetVT.SimpleTy) {
14655 case MVT::v8i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Op0, imm1);
14656 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Op0, imm1);
14657 default: return Register();
14658}
14659}
14660
14661Register fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14662 switch (VT.SimpleTy) {
14663 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1);
14664 default: return Register();
14665 }
14666}
14667
14668// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14669
14670Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, Register Op0, uint64_t imm1) {
14671 if (RetVT.SimpleTy != MVT::i32)
14672 return Register();
14673 if ((Subtarget->isNeonAvailable())) {
14674 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi8, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14675 }
14676 return Register();
14677}
14678
14679Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14680 switch (VT.SimpleTy) {
14681 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1);
14682 default: return Register();
14683 }
14684}
14685
14686// Top-level FastEmit function.
14687
14688Register fastEmit_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14689 switch (Opcode) {
14690 case AArch64ISD::DUPLANE8: return fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1);
14691 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1);
14692 default: return Register();
14693 }
14694}
14695
14696// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14697
14698Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14699 if (RetVT.SimpleTy != MVT::i32)
14700 return Register();
14701 if ((Subtarget->hasNEON())) {
14702 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi8_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14703 }
14704 return Register();
14705}
14706
14707Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14708 if (RetVT.SimpleTy != MVT::i32)
14709 return Register();
14710 if ((Subtarget->hasNEON())) {
14711 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi16_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14712 }
14713 return Register();
14714}
14715
14716Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14717 if (RetVT.SimpleTy != MVT::i32)
14718 return Register();
14719 if ((Subtarget->hasNEON())) {
14720 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi32_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14721 }
14722 return Register();
14723}
14724
14725Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14726 if (RetVT.SimpleTy != MVT::i64)
14727 return Register();
14728 if ((Subtarget->hasNEON())) {
14729 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi64_idx0, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14730 }
14731 return Register();
14732}
14733
14734Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14735 switch (VT.SimpleTy) {
14736 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14737 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14738 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14739 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14740 default: return Register();
14741 }
14742}
14743
14744// Top-level FastEmit function.
14745
14746Register fastEmit_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14747 switch (Opcode) {
14748 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(VT, RetVT, Op0, imm1);
14749 default: return Register();
14750 }
14751}
14752
14753// FastEmit functions for ISD::SMAX.
14754
14755Register fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14756 if (RetVT.SimpleTy != MVT::i32)
14757 return Register();
14758 if ((Subtarget->hasCSSC())) {
14759 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMAXWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14760 }
14761 return Register();
14762}
14763
14764Register fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14765 switch (VT.SimpleTy) {
14766 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1);
14767 default: return Register();
14768 }
14769}
14770
14771// FastEmit functions for ISD::SMIN.
14772
14773Register fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14774 if (RetVT.SimpleTy != MVT::i32)
14775 return Register();
14776 if ((Subtarget->hasCSSC())) {
14777 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMINWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14778 }
14779 return Register();
14780}
14781
14782Register fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14783 switch (VT.SimpleTy) {
14784 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1);
14785 default: return Register();
14786 }
14787}
14788
14789// Top-level FastEmit function.
14790
14791Register fastEmit_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14792 switch (Opcode) {
14793 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1);
14794 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1);
14795 default: return Register();
14796 }
14797}
14798
14799// FastEmit functions for ISD::SMAX.
14800
14801Register fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14802 if (RetVT.SimpleTy != MVT::i64)
14803 return Register();
14804 if ((Subtarget->hasCSSC())) {
14805 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMAXXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14806 }
14807 return Register();
14808}
14809
14810Register fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14811 switch (VT.SimpleTy) {
14812 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1);
14813 default: return Register();
14814 }
14815}
14816
14817// FastEmit functions for ISD::SMIN.
14818
14819Register fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14820 if (RetVT.SimpleTy != MVT::i64)
14821 return Register();
14822 if ((Subtarget->hasCSSC())) {
14823 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMINXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14824 }
14825 return Register();
14826}
14827
14828Register fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14829 switch (VT.SimpleTy) {
14830 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1);
14831 default: return Register();
14832 }
14833}
14834
14835// Top-level FastEmit function.
14836
14837Register fastEmit_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14838 switch (Opcode) {
14839 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1);
14840 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1);
14841 default: return Register();
14842 }
14843}
14844
14845// FastEmit functions for ISD::UMAX.
14846
14847Register fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14848 if (RetVT.SimpleTy != MVT::i32)
14849 return Register();
14850 if ((Subtarget->hasCSSC())) {
14851 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMAXWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14852 }
14853 return Register();
14854}
14855
14856Register fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14857 switch (VT.SimpleTy) {
14858 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1);
14859 default: return Register();
14860 }
14861}
14862
14863// FastEmit functions for ISD::UMIN.
14864
14865Register fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14866 if (RetVT.SimpleTy != MVT::i32)
14867 return Register();
14868 if ((Subtarget->hasCSSC())) {
14869 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMINWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14870 }
14871 return Register();
14872}
14873
14874Register fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14875 switch (VT.SimpleTy) {
14876 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1);
14877 default: return Register();
14878 }
14879}
14880
14881// Top-level FastEmit function.
14882
14883Register fastEmit_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14884 switch (Opcode) {
14885 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1);
14886 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1);
14887 default: return Register();
14888 }
14889}
14890
14891// FastEmit functions for ISD::UMAX.
14892
14893Register fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14894 if (RetVT.SimpleTy != MVT::i64)
14895 return Register();
14896 if ((Subtarget->hasCSSC())) {
14897 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMAXXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14898 }
14899 return Register();
14900}
14901
14902Register fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14903 switch (VT.SimpleTy) {
14904 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1);
14905 default: return Register();
14906 }
14907}
14908
14909// FastEmit functions for ISD::UMIN.
14910
14911Register fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14912 if (RetVT.SimpleTy != MVT::i64)
14913 return Register();
14914 if ((Subtarget->hasCSSC())) {
14915 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMINXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14916 }
14917 return Register();
14918}
14919
14920Register fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14921 switch (VT.SimpleTy) {
14922 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1);
14923 default: return Register();
14924 }
14925}
14926
14927// Top-level FastEmit function.
14928
14929Register fastEmit_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14930 switch (Opcode) {
14931 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1);
14932 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1);
14933 default: return Register();
14934 }
14935}
14936
14937// FastEmit functions for ISD::Constant.
14938
14939Register fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
14940 if (RetVT.SimpleTy != MVT::i32)
14941 return Register();
14942 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVi32imm, RC: &AArch64::GPR32RegClass, Imm: imm0);
14943}
14944
14945Register fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) {
14946 if (RetVT.SimpleTy != MVT::i64)
14947 return Register();
14948 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVi64imm, RC: &AArch64::GPR64RegClass, Imm: imm0);
14949}
14950
14951Register fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
14952 switch (VT.SimpleTy) {
14953 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
14954 case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0);
14955 default: return Register();
14956 }
14957}
14958
14959// Top-level FastEmit function.
14960
14961Register fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
14962 if (VT == MVT::i32 && Predicate_imm0_255(Imm: imm0))
14963 if (Register Reg = fastEmit_i_Predicate_imm0_255(VT, RetVT, Opcode, imm0))
14964 return Reg;
14965
14966 if (VT == MVT::i32 && Predicate_simm6_32b(Imm: imm0))
14967 if (Register Reg = fastEmit_i_Predicate_simm6_32b(VT, RetVT, Opcode, imm0))
14968 return Reg;
14969
14970 switch (Opcode) {
14971 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
14972 default: return Register();
14973 }
14974}
14975
14976// FastEmit functions for AArch64ISD::FMOV.
14977
14978Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(uint64_t imm0) {
14979 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
14980 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv4f16_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
14981 }
14982 return Register();
14983}
14984
14985Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(uint64_t imm0) {
14986 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
14987 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv8f16_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14988 }
14989 return Register();
14990}
14991
14992Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(uint64_t imm0) {
14993 if ((Subtarget->isNeonAvailable())) {
14994 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv2f32_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
14995 }
14996 return Register();
14997}
14998
14999Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(uint64_t imm0) {
15000 if ((Subtarget->isNeonAvailable())) {
15001 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv4f32_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
15002 }
15003 return Register();
15004}
15005
15006Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(uint64_t imm0) {
15007 if ((Subtarget->isNeonAvailable())) {
15008 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv2f64_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
15009 }
15010 return Register();
15011}
15012
15013Register fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
15014switch (RetVT.SimpleTy) {
15015 case MVT::v4f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(imm0);
15016 case MVT::v8f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(imm0);
15017 case MVT::v2f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(imm0);
15018 case MVT::v4f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(imm0);
15019 case MVT::v2f64: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(imm0);
15020 default: return Register();
15021}
15022}
15023
15024Register fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
15025 switch (VT.SimpleTy) {
15026 case MVT::i32: return fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
15027 default: return Register();
15028 }
15029}
15030
15031// FastEmit functions for AArch64ISD::MOVI.
15032
15033Register fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(uint64_t imm0) {
15034 if ((Subtarget->isNeonAvailable())) {
15035 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv8b_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
15036 }
15037 return Register();
15038}
15039
15040Register fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(uint64_t imm0) {
15041 if ((Subtarget->isNeonAvailable())) {
15042 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv16b_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
15043 }
15044 return Register();
15045}
15046
15047Register fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
15048switch (RetVT.SimpleTy) {
15049 case MVT::v8i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(imm0);
15050 case MVT::v16i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(imm0);
15051 default: return Register();
15052}
15053}
15054
15055Register fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
15056 switch (VT.SimpleTy) {
15057 case MVT::i32: return fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
15058 default: return Register();
15059 }
15060}
15061
15062// FastEmit functions for AArch64ISD::MOVIedit.
15063
15064Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(uint64_t imm0) {
15065 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVID, RC: &AArch64::FPR64RegClass, Imm: imm0);
15066}
15067
15068Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(uint64_t imm0) {
15069 if ((Subtarget->isNeonAvailable())) {
15070 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv2d_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
15071 }
15072 return Register();
15073}
15074
15075Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
15076switch (RetVT.SimpleTy) {
15077 case MVT::f64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(imm0);
15078 case MVT::v2i64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(imm0);
15079 default: return Register();
15080}
15081}
15082
15083Register fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
15084 switch (VT.SimpleTy) {
15085 case MVT::i32: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
15086 default: return Register();
15087 }
15088}
15089
15090// Top-level FastEmit function.
15091
15092Register fastEmit_i_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
15093 switch (Opcode) {
15094 case AArch64ISD::FMOV: return fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(VT, RetVT, imm0);
15095 case AArch64ISD::MOVI: return fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(VT, RetVT, imm0);
15096 case AArch64ISD::MOVIedit: return fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(VT, RetVT, imm0);
15097 default: return Register();
15098 }
15099}
15100
15101// FastEmit functions for AArch64ISD::RDSVL.
15102
15103Register fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(MVT RetVT, uint64_t imm0) {
15104 if (RetVT.SimpleTy != MVT::i64)
15105 return Register();
15106 if ((Subtarget->hasSME())) {
15107 return fastEmitInst_i(MachineInstOpcode: AArch64::RDSVLI_XI, RC: &AArch64::GPR64RegClass, Imm: imm0);
15108 }
15109 return Register();
15110}
15111
15112Register fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(MVT VT, MVT RetVT, uint64_t imm0) {
15113 switch (VT.SimpleTy) {
15114 case MVT::i32: return fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(RetVT, imm0);
15115 default: return Register();
15116 }
15117}
15118
15119// Top-level FastEmit function.
15120
15121Register fastEmit_i_Predicate_simm6_32b(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
15122 switch (Opcode) {
15123 case AArch64ISD::RDSVL: return fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(VT, RetVT, imm0);
15124 default: return Register();
15125 }
15126}
15127
15128