1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the AArch64 target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_imm0_31(int64_t Imm) {
12
13 return ((uint64_t)Imm) < 32;
14
15}
16static bool Predicate_imm0_63(int64_t Imm) {
17
18 return ((uint64_t)Imm) < 64;
19
20}
21static bool Predicate_imm32_0_31(int64_t Imm) {
22
23 return ((uint64_t)Imm) < 32;
24
25}
26static bool Predicate_tbz_imm0_31_diag(int64_t Imm) {
27
28 return (((uint32_t)Imm) < 32);
29
30}
31static bool Predicate_tbz_imm32_63(int64_t Imm) {
32
33 return (((uint32_t)Imm) > 31) && (((uint32_t)Imm) < 64);
34
35}
36static bool Predicate_VectorIndexD(int64_t Imm) {
37 return ((uint64_t)Imm) < 2;
38}
39static bool Predicate_VectorIndexS(int64_t Imm) {
40 return ((uint64_t)Imm) < 4;
41}
42static bool Predicate_VectorIndexH(int64_t Imm) {
43 return ((uint64_t)Imm) < 8;
44}
45static bool Predicate_VectorIndexB(int64_t Imm) {
46 return ((uint64_t)Imm) < 16;
47}
48static bool Predicate_VectorIndex0(int64_t Imm) {
49 return ((uint64_t)Imm) == 0;
50}
51static bool Predicate_imm0_255(int64_t Imm) {
52
53 return ((uint32_t)Imm) < 256;
54
55}
56static bool Predicate_simm8_32b(int64_t Imm) {
57 return Imm >= -128 && Imm < 128;
58}
59static bool Predicate_simm8_64b(int64_t Imm) {
60 return Imm >= -128 && Imm < 128;
61}
62static bool Predicate_uimm8_32b(int64_t Imm) {
63 return Imm >= 0 && Imm < 256;
64}
65static bool Predicate_uimm8_64b(int64_t Imm) {
66 return Imm >= 0 && Imm < 256;
67}
68static bool Predicate_simm6_32b(int64_t Imm) {
69 return Imm >= -32 && Imm < 32;
70}
71
72
73// FastEmit functions for AArch64ISD::ENTRY_PSTATE_SM.
74
75Register fastEmit_AArch64ISD_ENTRY_PSTATE_SM_MVT_i64_(MVT RetVT) {
76 if (RetVT.SimpleTy != MVT::i64)
77 return Register();
78 return fastEmitInst_(MachineInstOpcode: AArch64::EntryPStateSM, RC: &AArch64::GPR64RegClass);
79}
80
81Register fastEmit_AArch64ISD_ENTRY_PSTATE_SM_(MVT VT, MVT RetVT) {
82 switch (VT.SimpleTy) {
83 case MVT::i64: return fastEmit_AArch64ISD_ENTRY_PSTATE_SM_MVT_i64_(RetVT);
84 default: return Register();
85 }
86}
87
88// FastEmit functions for AArch64ISD::GET_SME_SAVE_SIZE.
89
90Register fastEmit_AArch64ISD_GET_SME_SAVE_SIZE_MVT_i64_(MVT RetVT) {
91 if (RetVT.SimpleTy != MVT::i64)
92 return Register();
93 return fastEmitInst_(MachineInstOpcode: AArch64::GetSMESaveSize, RC: &AArch64::GPR64RegClass);
94}
95
96Register fastEmit_AArch64ISD_GET_SME_SAVE_SIZE_(MVT VT, MVT RetVT) {
97 switch (VT.SimpleTy) {
98 case MVT::i64: return fastEmit_AArch64ISD_GET_SME_SAVE_SIZE_MVT_i64_(RetVT);
99 default: return Register();
100 }
101}
102
103// FastEmit functions for AArch64ISD::THREAD_POINTER.
104
105Register fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(MVT RetVT) {
106 if (RetVT.SimpleTy != MVT::i64)
107 return Register();
108 return fastEmitInst_(MachineInstOpcode: AArch64::MOVbaseTLS, RC: &AArch64::GPR64RegClass);
109}
110
111Register fastEmit_AArch64ISD_THREAD_POINTER_(MVT VT, MVT RetVT) {
112 switch (VT.SimpleTy) {
113 case MVT::i64: return fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(RetVT);
114 default: return Register();
115 }
116}
117
118// Top-level FastEmit function.
119
120Register fastEmit_(MVT VT, MVT RetVT, unsigned Opcode) override {
121 switch (Opcode) {
122 case AArch64ISD::ENTRY_PSTATE_SM: return fastEmit_AArch64ISD_ENTRY_PSTATE_SM_(VT, RetVT);
123 case AArch64ISD::GET_SME_SAVE_SIZE: return fastEmit_AArch64ISD_GET_SME_SAVE_SIZE_(VT, RetVT);
124 case AArch64ISD::THREAD_POINTER: return fastEmit_AArch64ISD_THREAD_POINTER_(VT, RetVT);
125 default: return Register();
126 }
127}
128
129// FastEmit functions for AArch64ISD::ALLOCATE_ZA_BUFFER.
130
131Register fastEmit_AArch64ISD_ALLOCATE_ZA_BUFFER_MVT_i64_r(MVT RetVT, Register Op0) {
132 if (RetVT.SimpleTy != MVT::i64)
133 return Register();
134 return fastEmitInst_r(MachineInstOpcode: AArch64::AllocateZABuffer, RC: &AArch64::GPR64spRegClass, Op0);
135}
136
137Register fastEmit_AArch64ISD_ALLOCATE_ZA_BUFFER_r(MVT VT, MVT RetVT, Register Op0) {
138 switch (VT.SimpleTy) {
139 case MVT::i64: return fastEmit_AArch64ISD_ALLOCATE_ZA_BUFFER_MVT_i64_r(RetVT, Op0);
140 default: return Register();
141 }
142}
143
144// FastEmit functions for AArch64ISD::ALLOC_SME_SAVE_BUFFER.
145
146Register fastEmit_AArch64ISD_ALLOC_SME_SAVE_BUFFER_MVT_i64_r(MVT RetVT, Register Op0) {
147 if (RetVT.SimpleTy != MVT::i64)
148 return Register();
149 return fastEmitInst_r(MachineInstOpcode: AArch64::AllocateSMESaveBuffer, RC: &AArch64::GPR64spRegClass, Op0);
150}
151
152Register fastEmit_AArch64ISD_ALLOC_SME_SAVE_BUFFER_r(MVT VT, MVT RetVT, Register Op0) {
153 switch (VT.SimpleTy) {
154 case MVT::i64: return fastEmit_AArch64ISD_ALLOC_SME_SAVE_BUFFER_MVT_i64_r(RetVT, Op0);
155 default: return Register();
156 }
157}
158
159// FastEmit functions for AArch64ISD::CALL.
160
161Register fastEmit_AArch64ISD_CALL_MVT_i64_r(MVT RetVT, Register Op0) {
162 if (RetVT.SimpleTy != MVT::isVoid)
163 return Register();
164 if (( MF->getSubtarget<AArch64Subtarget>().hardenSlsBlr() )) {
165 return fastEmitInst_r(MachineInstOpcode: AArch64::BLRNoIP, RC: &AArch64::GPR64noipRegClass, Op0);
166 }
167 if (( !MF->getSubtarget<AArch64Subtarget>().hardenSlsBlr() )) {
168 return fastEmitInst_r(MachineInstOpcode: AArch64::BLR, RC: &AArch64::GPR64RegClass, Op0);
169 }
170 return Register();
171}
172
173Register fastEmit_AArch64ISD_CALL_r(MVT VT, MVT RetVT, Register Op0) {
174 switch (VT.SimpleTy) {
175 case MVT::i64: return fastEmit_AArch64ISD_CALL_MVT_i64_r(RetVT, Op0);
176 default: return Register();
177 }
178}
179
180// FastEmit functions for AArch64ISD::COALESCER_BARRIER.
181
182Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_bf16_r(MVT RetVT, Register Op0) {
183 if (RetVT.SimpleTy != MVT::bf16)
184 return Register();
185 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR16, RC: &AArch64::FPR16RegClass, Op0);
186}
187
188Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f16_r(MVT RetVT, Register Op0) {
189 if (RetVT.SimpleTy != MVT::f16)
190 return Register();
191 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR16, RC: &AArch64::FPR16RegClass, Op0);
192}
193
194Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f32_r(MVT RetVT, Register Op0) {
195 if (RetVT.SimpleTy != MVT::f32)
196 return Register();
197 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR32, RC: &AArch64::FPR32RegClass, Op0);
198}
199
200Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f64_r(MVT RetVT, Register Op0) {
201 if (RetVT.SimpleTy != MVT::f64)
202 return Register();
203 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
204}
205
206Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f128_r(MVT RetVT, Register Op0) {
207 if (RetVT.SimpleTy != MVT::f128)
208 return Register();
209 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
210}
211
212Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i8_r(MVT RetVT, Register Op0) {
213 if (RetVT.SimpleTy != MVT::v8i8)
214 return Register();
215 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
216}
217
218Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v16i8_r(MVT RetVT, Register Op0) {
219 if (RetVT.SimpleTy != MVT::v16i8)
220 return Register();
221 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
222}
223
224Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i16_r(MVT RetVT, Register Op0) {
225 if (RetVT.SimpleTy != MVT::v4i16)
226 return Register();
227 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
228}
229
230Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i16_r(MVT RetVT, Register Op0) {
231 if (RetVT.SimpleTy != MVT::v8i16)
232 return Register();
233 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
234}
235
236Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i32_r(MVT RetVT, Register Op0) {
237 if (RetVT.SimpleTy != MVT::v2i32)
238 return Register();
239 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
240}
241
242Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i32_r(MVT RetVT, Register Op0) {
243 if (RetVT.SimpleTy != MVT::v4i32)
244 return Register();
245 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
246}
247
248Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1i64_r(MVT RetVT, Register Op0) {
249 if (RetVT.SimpleTy != MVT::v1i64)
250 return Register();
251 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
252}
253
254Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i64_r(MVT RetVT, Register Op0) {
255 if (RetVT.SimpleTy != MVT::v2i64)
256 return Register();
257 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
258}
259
260Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f16_r(MVT RetVT, Register Op0) {
261 if (RetVT.SimpleTy != MVT::v4f16)
262 return Register();
263 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
264}
265
266Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8f16_r(MVT RetVT, Register Op0) {
267 if (RetVT.SimpleTy != MVT::v8f16)
268 return Register();
269 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
270}
271
272Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4bf16_r(MVT RetVT, Register Op0) {
273 if (RetVT.SimpleTy != MVT::v4bf16)
274 return Register();
275 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
276}
277
278Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8bf16_r(MVT RetVT, Register Op0) {
279 if (RetVT.SimpleTy != MVT::v8bf16)
280 return Register();
281 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
282}
283
284Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f32_r(MVT RetVT, Register Op0) {
285 if (RetVT.SimpleTy != MVT::v2f32)
286 return Register();
287 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
288}
289
290Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f32_r(MVT RetVT, Register Op0) {
291 if (RetVT.SimpleTy != MVT::v4f32)
292 return Register();
293 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
294}
295
296Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1f64_r(MVT RetVT, Register Op0) {
297 if (RetVT.SimpleTy != MVT::v1f64)
298 return Register();
299 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
300}
301
302Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f64_r(MVT RetVT, Register Op0) {
303 if (RetVT.SimpleTy != MVT::v2f64)
304 return Register();
305 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
306}
307
308Register fastEmit_AArch64ISD_COALESCER_BARRIER_r(MVT VT, MVT RetVT, Register Op0) {
309 switch (VT.SimpleTy) {
310 case MVT::bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_bf16_r(RetVT, Op0);
311 case MVT::f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f16_r(RetVT, Op0);
312 case MVT::f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f32_r(RetVT, Op0);
313 case MVT::f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f64_r(RetVT, Op0);
314 case MVT::f128: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f128_r(RetVT, Op0);
315 case MVT::v8i8: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i8_r(RetVT, Op0);
316 case MVT::v16i8: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v16i8_r(RetVT, Op0);
317 case MVT::v4i16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i16_r(RetVT, Op0);
318 case MVT::v8i16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i16_r(RetVT, Op0);
319 case MVT::v2i32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i32_r(RetVT, Op0);
320 case MVT::v4i32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i32_r(RetVT, Op0);
321 case MVT::v1i64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1i64_r(RetVT, Op0);
322 case MVT::v2i64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i64_r(RetVT, Op0);
323 case MVT::v4f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f16_r(RetVT, Op0);
324 case MVT::v8f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8f16_r(RetVT, Op0);
325 case MVT::v4bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4bf16_r(RetVT, Op0);
326 case MVT::v8bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8bf16_r(RetVT, Op0);
327 case MVT::v2f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f32_r(RetVT, Op0);
328 case MVT::v4f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f32_r(RetVT, Op0);
329 case MVT::v1f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1f64_r(RetVT, Op0);
330 case MVT::v2f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f64_r(RetVT, Op0);
331 default: return Register();
332 }
333}
334
335// FastEmit functions for AArch64ISD::DUP.
336
337Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Register Op0) {
338 if ((Subtarget->isNeonAvailable())) {
339 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv8i8gpr, RC: &AArch64::FPR64RegClass, Op0);
340 }
341 return Register();
342}
343
344Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Register Op0) {
345 if ((Subtarget->isNeonAvailable())) {
346 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv16i8gpr, RC: &AArch64::FPR128RegClass, Op0);
347 }
348 return Register();
349}
350
351Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Register Op0) {
352 if ((Subtarget->isNeonAvailable())) {
353 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv4i16gpr, RC: &AArch64::FPR64RegClass, Op0);
354 }
355 return Register();
356}
357
358Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Register Op0) {
359 if ((Subtarget->isNeonAvailable())) {
360 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv8i16gpr, RC: &AArch64::FPR128RegClass, Op0);
361 }
362 return Register();
363}
364
365Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Register Op0) {
366 if ((Subtarget->isNeonAvailable())) {
367 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv2i32gpr, RC: &AArch64::FPR64RegClass, Op0);
368 }
369 return Register();
370}
371
372Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Register Op0) {
373 if ((Subtarget->isNeonAvailable())) {
374 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv4i32gpr, RC: &AArch64::FPR128RegClass, Op0);
375 }
376 return Register();
377}
378
379Register fastEmit_AArch64ISD_DUP_MVT_i32_r(MVT RetVT, Register Op0) {
380switch (RetVT.SimpleTy) {
381 case MVT::v8i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Op0);
382 case MVT::v16i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Op0);
383 case MVT::v4i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Op0);
384 case MVT::v8i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Op0);
385 case MVT::v2i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Op0);
386 case MVT::v4i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Op0);
387 default: return Register();
388}
389}
390
391Register fastEmit_AArch64ISD_DUP_MVT_i64_r(MVT RetVT, Register Op0) {
392 if (RetVT.SimpleTy != MVT::v2i64)
393 return Register();
394 if ((Subtarget->isNeonAvailable())) {
395 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv2i64gpr, RC: &AArch64::FPR128RegClass, Op0);
396 }
397 return Register();
398}
399
400Register fastEmit_AArch64ISD_DUP_r(MVT VT, MVT RetVT, Register Op0) {
401 switch (VT.SimpleTy) {
402 case MVT::i32: return fastEmit_AArch64ISD_DUP_MVT_i32_r(RetVT, Op0);
403 case MVT::i64: return fastEmit_AArch64ISD_DUP_MVT_i64_r(RetVT, Op0);
404 default: return Register();
405 }
406}
407
408// FastEmit functions for AArch64ISD::FCVTXN.
409
410Register fastEmit_AArch64ISD_FCVTXN_MVT_f64_r(MVT RetVT, Register Op0) {
411 if (RetVT.SimpleTy != MVT::f32)
412 return Register();
413 if ((Subtarget->isNeonAvailable())) {
414 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTXNv1i64, RC: &AArch64::FPR32RegClass, Op0);
415 }
416 return Register();
417}
418
419Register fastEmit_AArch64ISD_FCVTXN_MVT_v2f64_r(MVT RetVT, Register Op0) {
420 if (RetVT.SimpleTy != MVT::v2f32)
421 return Register();
422 if ((Subtarget->isNeonAvailable())) {
423 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTXNv2f32, RC: &AArch64::FPR64RegClass, Op0);
424 }
425 return Register();
426}
427
428Register fastEmit_AArch64ISD_FCVTXN_r(MVT VT, MVT RetVT, Register Op0) {
429 switch (VT.SimpleTy) {
430 case MVT::f64: return fastEmit_AArch64ISD_FCVTXN_MVT_f64_r(RetVT, Op0);
431 case MVT::v2f64: return fastEmit_AArch64ISD_FCVTXN_MVT_v2f64_r(RetVT, Op0);
432 default: return Register();
433 }
434}
435
436// FastEmit functions for AArch64ISD::FRECPE.
437
438Register fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(MVT RetVT, Register Op0) {
439 if (RetVT.SimpleTy != MVT::v2f32)
440 return Register();
441 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv2f32, RC: &AArch64::FPR64RegClass, Op0);
442}
443
444Register fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(MVT RetVT, Register Op0) {
445 if (RetVT.SimpleTy != MVT::v4f32)
446 return Register();
447 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv4f32, RC: &AArch64::FPR128RegClass, Op0);
448}
449
450Register fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(MVT RetVT, Register Op0) {
451 if (RetVT.SimpleTy != MVT::v2f64)
452 return Register();
453 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv2f64, RC: &AArch64::FPR128RegClass, Op0);
454}
455
456Register fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
457 if (RetVT.SimpleTy != MVT::nxv8f16)
458 return Register();
459 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
460 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
461 }
462 return Register();
463}
464
465Register fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
466 if (RetVT.SimpleTy != MVT::nxv4f32)
467 return Register();
468 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
469 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
470 }
471 return Register();
472}
473
474Register fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
475 if (RetVT.SimpleTy != MVT::nxv2f64)
476 return Register();
477 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
478 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
479 }
480 return Register();
481}
482
483Register fastEmit_AArch64ISD_FRECPE_r(MVT VT, MVT RetVT, Register Op0) {
484 switch (VT.SimpleTy) {
485 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(RetVT, Op0);
486 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(RetVT, Op0);
487 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(RetVT, Op0);
488 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(RetVT, Op0);
489 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(RetVT, Op0);
490 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(RetVT, Op0);
491 default: return Register();
492 }
493}
494
495// FastEmit functions for AArch64ISD::FRSQRTE.
496
497Register fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(MVT RetVT, Register Op0) {
498 if (RetVT.SimpleTy != MVT::v2f32)
499 return Register();
500 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv2f32, RC: &AArch64::FPR64RegClass, Op0);
501}
502
503Register fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(MVT RetVT, Register Op0) {
504 if (RetVT.SimpleTy != MVT::v4f32)
505 return Register();
506 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv4f32, RC: &AArch64::FPR128RegClass, Op0);
507}
508
509Register fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(MVT RetVT, Register Op0) {
510 if (RetVT.SimpleTy != MVT::v2f64)
511 return Register();
512 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv2f64, RC: &AArch64::FPR128RegClass, Op0);
513}
514
515Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
516 if (RetVT.SimpleTy != MVT::nxv8f16)
517 return Register();
518 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
519 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
520 }
521 return Register();
522}
523
524Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
525 if (RetVT.SimpleTy != MVT::nxv4f32)
526 return Register();
527 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
528 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
529 }
530 return Register();
531}
532
533Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
534 if (RetVT.SimpleTy != MVT::nxv2f64)
535 return Register();
536 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
537 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
538 }
539 return Register();
540}
541
542Register fastEmit_AArch64ISD_FRSQRTE_r(MVT VT, MVT RetVT, Register Op0) {
543 switch (VT.SimpleTy) {
544 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(RetVT, Op0);
545 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(RetVT, Op0);
546 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(RetVT, Op0);
547 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(RetVT, Op0);
548 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(RetVT, Op0);
549 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(RetVT, Op0);
550 default: return Register();
551 }
552}
553
554// FastEmit functions for AArch64ISD::PROBED_ALLOCA.
555
556Register fastEmit_AArch64ISD_PROBED_ALLOCA_MVT_i64_r(MVT RetVT, Register Op0) {
557 if (RetVT.SimpleTy != MVT::isVoid)
558 return Register();
559 return fastEmitInst_r(MachineInstOpcode: AArch64::PROBED_STACKALLOC_DYN, RC: &AArch64::GPR64commonRegClass, Op0);
560}
561
562Register fastEmit_AArch64ISD_PROBED_ALLOCA_r(MVT VT, MVT RetVT, Register Op0) {
563 switch (VT.SimpleTy) {
564 case MVT::i64: return fastEmit_AArch64ISD_PROBED_ALLOCA_MVT_i64_r(RetVT, Op0);
565 default: return Register();
566 }
567}
568
569// FastEmit functions for AArch64ISD::REV16.
570
571Register fastEmit_AArch64ISD_REV16_MVT_i32_r(MVT RetVT, Register Op0) {
572 if (RetVT.SimpleTy != MVT::i32)
573 return Register();
574 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16Wr, RC: &AArch64::GPR32RegClass, Op0);
575}
576
577Register fastEmit_AArch64ISD_REV16_MVT_i64_r(MVT RetVT, Register Op0) {
578 if (RetVT.SimpleTy != MVT::i64)
579 return Register();
580 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16Xr, RC: &AArch64::GPR64RegClass, Op0);
581}
582
583Register fastEmit_AArch64ISD_REV16_MVT_v8i8_r(MVT RetVT, Register Op0) {
584 if (RetVT.SimpleTy != MVT::v8i8)
585 return Register();
586 if ((Subtarget->isNeonAvailable())) {
587 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
588 }
589 return Register();
590}
591
592Register fastEmit_AArch64ISD_REV16_MVT_v16i8_r(MVT RetVT, Register Op0) {
593 if (RetVT.SimpleTy != MVT::v16i8)
594 return Register();
595 if ((Subtarget->isNeonAvailable())) {
596 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
597 }
598 return Register();
599}
600
601Register fastEmit_AArch64ISD_REV16_r(MVT VT, MVT RetVT, Register Op0) {
602 switch (VT.SimpleTy) {
603 case MVT::i32: return fastEmit_AArch64ISD_REV16_MVT_i32_r(RetVT, Op0);
604 case MVT::i64: return fastEmit_AArch64ISD_REV16_MVT_i64_r(RetVT, Op0);
605 case MVT::v8i8: return fastEmit_AArch64ISD_REV16_MVT_v8i8_r(RetVT, Op0);
606 case MVT::v16i8: return fastEmit_AArch64ISD_REV16_MVT_v16i8_r(RetVT, Op0);
607 default: return Register();
608 }
609}
610
611// FastEmit functions for AArch64ISD::REV32.
612
613Register fastEmit_AArch64ISD_REV32_MVT_v8i8_r(MVT RetVT, Register Op0) {
614 if (RetVT.SimpleTy != MVT::v8i8)
615 return Register();
616 if ((Subtarget->isNeonAvailable())) {
617 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
618 }
619 return Register();
620}
621
622Register fastEmit_AArch64ISD_REV32_MVT_v16i8_r(MVT RetVT, Register Op0) {
623 if (RetVT.SimpleTy != MVT::v16i8)
624 return Register();
625 if ((Subtarget->isNeonAvailable())) {
626 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
627 }
628 return Register();
629}
630
631Register fastEmit_AArch64ISD_REV32_MVT_v4i16_r(MVT RetVT, Register Op0) {
632 if (RetVT.SimpleTy != MVT::v4i16)
633 return Register();
634 if ((Subtarget->isNeonAvailable())) {
635 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
636 }
637 return Register();
638}
639
640Register fastEmit_AArch64ISD_REV32_MVT_v8i16_r(MVT RetVT, Register Op0) {
641 if (RetVT.SimpleTy != MVT::v8i16)
642 return Register();
643 if ((Subtarget->isNeonAvailable())) {
644 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
645 }
646 return Register();
647}
648
649Register fastEmit_AArch64ISD_REV32_MVT_v4f16_r(MVT RetVT, Register Op0) {
650 if (RetVT.SimpleTy != MVT::v4f16)
651 return Register();
652 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
653}
654
655Register fastEmit_AArch64ISD_REV32_MVT_v8f16_r(MVT RetVT, Register Op0) {
656 if (RetVT.SimpleTy != MVT::v8f16)
657 return Register();
658 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
659}
660
661Register fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(MVT RetVT, Register Op0) {
662 if (RetVT.SimpleTy != MVT::v4bf16)
663 return Register();
664 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
665}
666
667Register fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(MVT RetVT, Register Op0) {
668 if (RetVT.SimpleTy != MVT::v8bf16)
669 return Register();
670 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
671}
672
673Register fastEmit_AArch64ISD_REV32_r(MVT VT, MVT RetVT, Register Op0) {
674 switch (VT.SimpleTy) {
675 case MVT::v8i8: return fastEmit_AArch64ISD_REV32_MVT_v8i8_r(RetVT, Op0);
676 case MVT::v16i8: return fastEmit_AArch64ISD_REV32_MVT_v16i8_r(RetVT, Op0);
677 case MVT::v4i16: return fastEmit_AArch64ISD_REV32_MVT_v4i16_r(RetVT, Op0);
678 case MVT::v8i16: return fastEmit_AArch64ISD_REV32_MVT_v8i16_r(RetVT, Op0);
679 case MVT::v4f16: return fastEmit_AArch64ISD_REV32_MVT_v4f16_r(RetVT, Op0);
680 case MVT::v8f16: return fastEmit_AArch64ISD_REV32_MVT_v8f16_r(RetVT, Op0);
681 case MVT::v4bf16: return fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(RetVT, Op0);
682 case MVT::v8bf16: return fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(RetVT, Op0);
683 default: return Register();
684 }
685}
686
687// FastEmit functions for AArch64ISD::REV64.
688
689Register fastEmit_AArch64ISD_REV64_MVT_v8i8_r(MVT RetVT, Register Op0) {
690 if (RetVT.SimpleTy != MVT::v8i8)
691 return Register();
692 if ((Subtarget->isNeonAvailable())) {
693 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
694 }
695 return Register();
696}
697
698Register fastEmit_AArch64ISD_REV64_MVT_v16i8_r(MVT RetVT, Register Op0) {
699 if (RetVT.SimpleTy != MVT::v16i8)
700 return Register();
701 if ((Subtarget->isNeonAvailable())) {
702 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
703 }
704 return Register();
705}
706
707Register fastEmit_AArch64ISD_REV64_MVT_v4i16_r(MVT RetVT, Register Op0) {
708 if (RetVT.SimpleTy != MVT::v4i16)
709 return Register();
710 if ((Subtarget->isNeonAvailable())) {
711 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
712 }
713 return Register();
714}
715
716Register fastEmit_AArch64ISD_REV64_MVT_v8i16_r(MVT RetVT, Register Op0) {
717 if (RetVT.SimpleTy != MVT::v8i16)
718 return Register();
719 if ((Subtarget->isNeonAvailable())) {
720 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
721 }
722 return Register();
723}
724
725Register fastEmit_AArch64ISD_REV64_MVT_v2i32_r(MVT RetVT, Register Op0) {
726 if (RetVT.SimpleTy != MVT::v2i32)
727 return Register();
728 if ((Subtarget->isNeonAvailable())) {
729 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
730 }
731 return Register();
732}
733
734Register fastEmit_AArch64ISD_REV64_MVT_v4i32_r(MVT RetVT, Register Op0) {
735 if (RetVT.SimpleTy != MVT::v4i32)
736 return Register();
737 if ((Subtarget->isNeonAvailable())) {
738 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
739 }
740 return Register();
741}
742
743Register fastEmit_AArch64ISD_REV64_MVT_v4f16_r(MVT RetVT, Register Op0) {
744 if (RetVT.SimpleTy != MVT::v4f16)
745 return Register();
746 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
747}
748
749Register fastEmit_AArch64ISD_REV64_MVT_v8f16_r(MVT RetVT, Register Op0) {
750 if (RetVT.SimpleTy != MVT::v8f16)
751 return Register();
752 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
753}
754
755Register fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(MVT RetVT, Register Op0) {
756 if (RetVT.SimpleTy != MVT::v4bf16)
757 return Register();
758 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
759}
760
761Register fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(MVT RetVT, Register Op0) {
762 if (RetVT.SimpleTy != MVT::v8bf16)
763 return Register();
764 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
765}
766
767Register fastEmit_AArch64ISD_REV64_MVT_v2f32_r(MVT RetVT, Register Op0) {
768 if (RetVT.SimpleTy != MVT::v2f32)
769 return Register();
770 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
771}
772
773Register fastEmit_AArch64ISD_REV64_MVT_v4f32_r(MVT RetVT, Register Op0) {
774 if (RetVT.SimpleTy != MVT::v4f32)
775 return Register();
776 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
777}
778
779Register fastEmit_AArch64ISD_REV64_r(MVT VT, MVT RetVT, Register Op0) {
780 switch (VT.SimpleTy) {
781 case MVT::v8i8: return fastEmit_AArch64ISD_REV64_MVT_v8i8_r(RetVT, Op0);
782 case MVT::v16i8: return fastEmit_AArch64ISD_REV64_MVT_v16i8_r(RetVT, Op0);
783 case MVT::v4i16: return fastEmit_AArch64ISD_REV64_MVT_v4i16_r(RetVT, Op0);
784 case MVT::v8i16: return fastEmit_AArch64ISD_REV64_MVT_v8i16_r(RetVT, Op0);
785 case MVT::v2i32: return fastEmit_AArch64ISD_REV64_MVT_v2i32_r(RetVT, Op0);
786 case MVT::v4i32: return fastEmit_AArch64ISD_REV64_MVT_v4i32_r(RetVT, Op0);
787 case MVT::v4f16: return fastEmit_AArch64ISD_REV64_MVT_v4f16_r(RetVT, Op0);
788 case MVT::v8f16: return fastEmit_AArch64ISD_REV64_MVT_v8f16_r(RetVT, Op0);
789 case MVT::v4bf16: return fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(RetVT, Op0);
790 case MVT::v8bf16: return fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(RetVT, Op0);
791 case MVT::v2f32: return fastEmit_AArch64ISD_REV64_MVT_v2f32_r(RetVT, Op0);
792 case MVT::v4f32: return fastEmit_AArch64ISD_REV64_MVT_v4f32_r(RetVT, Op0);
793 default: return Register();
794 }
795}
796
797// FastEmit functions for AArch64ISD::SADDLP.
798
799Register fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(MVT RetVT, Register Op0) {
800 if (RetVT.SimpleTy != MVT::v4i16)
801 return Register();
802 if ((Subtarget->isNeonAvailable())) {
803 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv8i8_v4i16, RC: &AArch64::FPR64RegClass, Op0);
804 }
805 return Register();
806}
807
808Register fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(MVT RetVT, Register Op0) {
809 if (RetVT.SimpleTy != MVT::v8i16)
810 return Register();
811 if ((Subtarget->isNeonAvailable())) {
812 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv16i8_v8i16, RC: &AArch64::FPR128RegClass, Op0);
813 }
814 return Register();
815}
816
817Register fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(MVT RetVT, Register Op0) {
818 if (RetVT.SimpleTy != MVT::v2i32)
819 return Register();
820 if ((Subtarget->isNeonAvailable())) {
821 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv4i16_v2i32, RC: &AArch64::FPR64RegClass, Op0);
822 }
823 return Register();
824}
825
826Register fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(MVT RetVT, Register Op0) {
827 if (RetVT.SimpleTy != MVT::v4i32)
828 return Register();
829 if ((Subtarget->isNeonAvailable())) {
830 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv8i16_v4i32, RC: &AArch64::FPR128RegClass, Op0);
831 }
832 return Register();
833}
834
835Register fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(MVT RetVT, Register Op0) {
836 if (RetVT.SimpleTy != MVT::v1i64)
837 return Register();
838 if ((Subtarget->isNeonAvailable())) {
839 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv2i32_v1i64, RC: &AArch64::FPR64RegClass, Op0);
840 }
841 return Register();
842}
843
844Register fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(MVT RetVT, Register Op0) {
845 if (RetVT.SimpleTy != MVT::v2i64)
846 return Register();
847 if ((Subtarget->isNeonAvailable())) {
848 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv4i32_v2i64, RC: &AArch64::FPR128RegClass, Op0);
849 }
850 return Register();
851}
852
853Register fastEmit_AArch64ISD_SADDLP_r(MVT VT, MVT RetVT, Register Op0) {
854 switch (VT.SimpleTy) {
855 case MVT::v8i8: return fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(RetVT, Op0);
856 case MVT::v16i8: return fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(RetVT, Op0);
857 case MVT::v4i16: return fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(RetVT, Op0);
858 case MVT::v8i16: return fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(RetVT, Op0);
859 case MVT::v2i32: return fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(RetVT, Op0);
860 case MVT::v4i32: return fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(RetVT, Op0);
861 default: return Register();
862 }
863}
864
865// FastEmit functions for AArch64ISD::SITOF.
866
867Register fastEmit_AArch64ISD_SITOF_MVT_f16_r(MVT RetVT, Register Op0) {
868 if (RetVT.SimpleTy != MVT::f16)
869 return Register();
870 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
871 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i16, RC: &AArch64::FPR16RegClass, Op0);
872 }
873 return Register();
874}
875
876Register fastEmit_AArch64ISD_SITOF_MVT_f32_r(MVT RetVT, Register Op0) {
877 if (RetVT.SimpleTy != MVT::f32)
878 return Register();
879 if ((Subtarget->hasNEON())) {
880 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i32, RC: &AArch64::FPR32RegClass, Op0);
881 }
882 return Register();
883}
884
885Register fastEmit_AArch64ISD_SITOF_MVT_f64_r(MVT RetVT, Register Op0) {
886 if (RetVT.SimpleTy != MVT::f64)
887 return Register();
888 if ((Subtarget->hasNEON())) {
889 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i64, RC: &AArch64::FPR64RegClass, Op0);
890 }
891 return Register();
892}
893
894Register fastEmit_AArch64ISD_SITOF_r(MVT VT, MVT RetVT, Register Op0) {
895 switch (VT.SimpleTy) {
896 case MVT::f16: return fastEmit_AArch64ISD_SITOF_MVT_f16_r(RetVT, Op0);
897 case MVT::f32: return fastEmit_AArch64ISD_SITOF_MVT_f32_r(RetVT, Op0);
898 case MVT::f64: return fastEmit_AArch64ISD_SITOF_MVT_f64_r(RetVT, Op0);
899 default: return Register();
900 }
901}
902
903// FastEmit functions for AArch64ISD::SUNPKHI.
904
905Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
906 if (RetVT.SimpleTy != MVT::nxv8i16)
907 return Register();
908 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
909 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
910 }
911 return Register();
912}
913
914Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
915 if (RetVT.SimpleTy != MVT::nxv4i32)
916 return Register();
917 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
918 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
919 }
920 return Register();
921}
922
923Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
924 if (RetVT.SimpleTy != MVT::nxv2i64)
925 return Register();
926 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
927 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
928 }
929 return Register();
930}
931
932Register fastEmit_AArch64ISD_SUNPKHI_r(MVT VT, MVT RetVT, Register Op0) {
933 switch (VT.SimpleTy) {
934 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(RetVT, Op0);
935 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(RetVT, Op0);
936 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(RetVT, Op0);
937 default: return Register();
938 }
939}
940
941// FastEmit functions for AArch64ISD::SUNPKLO.
942
943Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
944 if (RetVT.SimpleTy != MVT::nxv8i16)
945 return Register();
946 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
947 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
948 }
949 return Register();
950}
951
952Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
953 if (RetVT.SimpleTy != MVT::nxv4i32)
954 return Register();
955 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
956 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
957 }
958 return Register();
959}
960
961Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
962 if (RetVT.SimpleTy != MVT::nxv2i64)
963 return Register();
964 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
965 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
966 }
967 return Register();
968}
969
970Register fastEmit_AArch64ISD_SUNPKLO_r(MVT VT, MVT RetVT, Register Op0) {
971 switch (VT.SimpleTy) {
972 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(RetVT, Op0);
973 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(RetVT, Op0);
974 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(RetVT, Op0);
975 default: return Register();
976 }
977}
978
979// FastEmit functions for AArch64ISD::UADDLP.
980
981Register fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(MVT RetVT, Register Op0) {
982 if (RetVT.SimpleTy != MVT::v4i16)
983 return Register();
984 if ((Subtarget->isNeonAvailable())) {
985 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv8i8_v4i16, RC: &AArch64::FPR64RegClass, Op0);
986 }
987 return Register();
988}
989
990Register fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(MVT RetVT, Register Op0) {
991 if (RetVT.SimpleTy != MVT::v8i16)
992 return Register();
993 if ((Subtarget->isNeonAvailable())) {
994 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv16i8_v8i16, RC: &AArch64::FPR128RegClass, Op0);
995 }
996 return Register();
997}
998
999Register fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(MVT RetVT, Register Op0) {
1000 if (RetVT.SimpleTy != MVT::v2i32)
1001 return Register();
1002 if ((Subtarget->isNeonAvailable())) {
1003 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv4i16_v2i32, RC: &AArch64::FPR64RegClass, Op0);
1004 }
1005 return Register();
1006}
1007
1008Register fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(MVT RetVT, Register Op0) {
1009 if (RetVT.SimpleTy != MVT::v4i32)
1010 return Register();
1011 if ((Subtarget->isNeonAvailable())) {
1012 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv8i16_v4i32, RC: &AArch64::FPR128RegClass, Op0);
1013 }
1014 return Register();
1015}
1016
1017Register fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(MVT RetVT, Register Op0) {
1018 if (RetVT.SimpleTy != MVT::v1i64)
1019 return Register();
1020 if ((Subtarget->isNeonAvailable())) {
1021 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv2i32_v1i64, RC: &AArch64::FPR64RegClass, Op0);
1022 }
1023 return Register();
1024}
1025
1026Register fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(MVT RetVT, Register Op0) {
1027 if (RetVT.SimpleTy != MVT::v2i64)
1028 return Register();
1029 if ((Subtarget->isNeonAvailable())) {
1030 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv4i32_v2i64, RC: &AArch64::FPR128RegClass, Op0);
1031 }
1032 return Register();
1033}
1034
1035Register fastEmit_AArch64ISD_UADDLP_r(MVT VT, MVT RetVT, Register Op0) {
1036 switch (VT.SimpleTy) {
1037 case MVT::v8i8: return fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(RetVT, Op0);
1038 case MVT::v16i8: return fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(RetVT, Op0);
1039 case MVT::v4i16: return fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(RetVT, Op0);
1040 case MVT::v8i16: return fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(RetVT, Op0);
1041 case MVT::v2i32: return fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(RetVT, Op0);
1042 case MVT::v4i32: return fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(RetVT, Op0);
1043 default: return Register();
1044 }
1045}
1046
1047// FastEmit functions for AArch64ISD::UITOF.
1048
1049Register fastEmit_AArch64ISD_UITOF_MVT_f16_r(MVT RetVT, Register Op0) {
1050 if (RetVT.SimpleTy != MVT::f16)
1051 return Register();
1052 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
1053 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i16, RC: &AArch64::FPR16RegClass, Op0);
1054 }
1055 return Register();
1056}
1057
1058Register fastEmit_AArch64ISD_UITOF_MVT_f32_r(MVT RetVT, Register Op0) {
1059 if (RetVT.SimpleTy != MVT::f32)
1060 return Register();
1061 if ((Subtarget->hasNEON())) {
1062 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i32, RC: &AArch64::FPR32RegClass, Op0);
1063 }
1064 return Register();
1065}
1066
1067Register fastEmit_AArch64ISD_UITOF_MVT_f64_r(MVT RetVT, Register Op0) {
1068 if (RetVT.SimpleTy != MVT::f64)
1069 return Register();
1070 if ((Subtarget->hasNEON())) {
1071 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i64, RC: &AArch64::FPR64RegClass, Op0);
1072 }
1073 return Register();
1074}
1075
1076Register fastEmit_AArch64ISD_UITOF_r(MVT VT, MVT RetVT, Register Op0) {
1077 switch (VT.SimpleTy) {
1078 case MVT::f16: return fastEmit_AArch64ISD_UITOF_MVT_f16_r(RetVT, Op0);
1079 case MVT::f32: return fastEmit_AArch64ISD_UITOF_MVT_f32_r(RetVT, Op0);
1080 case MVT::f64: return fastEmit_AArch64ISD_UITOF_MVT_f64_r(RetVT, Op0);
1081 default: return Register();
1082 }
1083}
1084
1085// FastEmit functions for AArch64ISD::UUNPKHI.
1086
1087Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
1088 if (RetVT.SimpleTy != MVT::nxv8i16)
1089 return Register();
1090 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1091 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1092 }
1093 return Register();
1094}
1095
1096Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
1097 if (RetVT.SimpleTy != MVT::nxv4i32)
1098 return Register();
1099 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1100 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1101 }
1102 return Register();
1103}
1104
1105Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
1106 if (RetVT.SimpleTy != MVT::nxv2i64)
1107 return Register();
1108 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1109 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1110 }
1111 return Register();
1112}
1113
1114Register fastEmit_AArch64ISD_UUNPKHI_r(MVT VT, MVT RetVT, Register Op0) {
1115 switch (VT.SimpleTy) {
1116 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(RetVT, Op0);
1117 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(RetVT, Op0);
1118 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(RetVT, Op0);
1119 default: return Register();
1120 }
1121}
1122
1123// FastEmit functions for AArch64ISD::UUNPKLO.
1124
1125Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
1126 if (RetVT.SimpleTy != MVT::nxv8i16)
1127 return Register();
1128 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1129 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1130 }
1131 return Register();
1132}
1133
1134Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
1135 if (RetVT.SimpleTy != MVT::nxv4i32)
1136 return Register();
1137 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1138 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1139 }
1140 return Register();
1141}
1142
1143Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
1144 if (RetVT.SimpleTy != MVT::nxv2i64)
1145 return Register();
1146 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1147 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1148 }
1149 return Register();
1150}
1151
1152Register fastEmit_AArch64ISD_UUNPKLO_r(MVT VT, MVT RetVT, Register Op0) {
1153 switch (VT.SimpleTy) {
1154 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(RetVT, Op0);
1155 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(RetVT, Op0);
1156 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(RetVT, Op0);
1157 default: return Register();
1158 }
1159}
1160
1161// FastEmit functions for ISD::ABS.
1162
1163Register fastEmit_ISD_ABS_MVT_i32_r(MVT RetVT, Register Op0) {
1164 if (RetVT.SimpleTy != MVT::i32)
1165 return Register();
1166 if ((Subtarget->hasCSSC())) {
1167 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSWr, RC: &AArch64::GPR32RegClass, Op0);
1168 }
1169 return Register();
1170}
1171
1172Register fastEmit_ISD_ABS_MVT_i64_r(MVT RetVT, Register Op0) {
1173 if (RetVT.SimpleTy != MVT::i64)
1174 return Register();
1175 if ((!Subtarget->hasCSSC())) {
1176 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
1177 }
1178 if ((Subtarget->hasCSSC())) {
1179 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSXr, RC: &AArch64::GPR64RegClass, Op0);
1180 }
1181 return Register();
1182}
1183
1184Register fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, Register Op0) {
1185 if (RetVT.SimpleTy != MVT::v8i8)
1186 return Register();
1187 if ((Subtarget->isNeonAvailable())) {
1188 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv8i8, RC: &AArch64::FPR64RegClass, Op0);
1189 }
1190 return Register();
1191}
1192
1193Register fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, Register Op0) {
1194 if (RetVT.SimpleTy != MVT::v16i8)
1195 return Register();
1196 if ((Subtarget->isNeonAvailable())) {
1197 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv16i8, RC: &AArch64::FPR128RegClass, Op0);
1198 }
1199 return Register();
1200}
1201
1202Register fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, Register Op0) {
1203 if (RetVT.SimpleTy != MVT::v4i16)
1204 return Register();
1205 if ((Subtarget->isNeonAvailable())) {
1206 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv4i16, RC: &AArch64::FPR64RegClass, Op0);
1207 }
1208 return Register();
1209}
1210
1211Register fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, Register Op0) {
1212 if (RetVT.SimpleTy != MVT::v8i16)
1213 return Register();
1214 if ((Subtarget->isNeonAvailable())) {
1215 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv8i16, RC: &AArch64::FPR128RegClass, Op0);
1216 }
1217 return Register();
1218}
1219
1220Register fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, Register Op0) {
1221 if (RetVT.SimpleTy != MVT::v2i32)
1222 return Register();
1223 if ((Subtarget->isNeonAvailable())) {
1224 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv2i32, RC: &AArch64::FPR64RegClass, Op0);
1225 }
1226 return Register();
1227}
1228
1229Register fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, Register Op0) {
1230 if (RetVT.SimpleTy != MVT::v4i32)
1231 return Register();
1232 if ((Subtarget->isNeonAvailable())) {
1233 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv4i32, RC: &AArch64::FPR128RegClass, Op0);
1234 }
1235 return Register();
1236}
1237
1238Register fastEmit_ISD_ABS_MVT_v1i64_r(MVT RetVT, Register Op0) {
1239 if (RetVT.SimpleTy != MVT::v1i64)
1240 return Register();
1241 if ((Subtarget->isNeonAvailable())) {
1242 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
1243 }
1244 return Register();
1245}
1246
1247Register fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, Register Op0) {
1248 if (RetVT.SimpleTy != MVT::v2i64)
1249 return Register();
1250 if ((Subtarget->isNeonAvailable())) {
1251 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv2i64, RC: &AArch64::FPR128RegClass, Op0);
1252 }
1253 return Register();
1254}
1255
1256Register fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, Register Op0) {
1257 switch (VT.SimpleTy) {
1258 case MVT::i32: return fastEmit_ISD_ABS_MVT_i32_r(RetVT, Op0);
1259 case MVT::i64: return fastEmit_ISD_ABS_MVT_i64_r(RetVT, Op0);
1260 case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0);
1261 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
1262 case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0);
1263 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
1264 case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0);
1265 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
1266 case MVT::v1i64: return fastEmit_ISD_ABS_MVT_v1i64_r(RetVT, Op0);
1267 case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0);
1268 default: return Register();
1269 }
1270}
1271
1272// FastEmit functions for ISD::BITCAST.
1273
1274Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Register Op0) {
1275 if ((!Subtarget->isLittleEndian())) {
1276 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1277 }
1278 return Register();
1279}
1280
1281Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Register Op0) {
1282 if ((!Subtarget->isLittleEndian())) {
1283 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1284 }
1285 return Register();
1286}
1287
1288Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Register Op0) {
1289 if ((!Subtarget->isLittleEndian())) {
1290 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1291 }
1292 return Register();
1293}
1294
1295Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Register Op0) {
1296 if ((!Subtarget->isLittleEndian())) {
1297 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1298 }
1299 return Register();
1300}
1301
1302Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Register Op0) {
1303 if ((!Subtarget->isLittleEndian())) {
1304 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1305 }
1306 return Register();
1307}
1308
1309Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Register Op0) {
1310 if ((!Subtarget->isLittleEndian())) {
1311 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1312 }
1313 return Register();
1314}
1315
1316Register fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, Register Op0) {
1317switch (RetVT.SimpleTy) {
1318 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0);
1319 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0);
1320 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0);
1321 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0);
1322 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Op0);
1323 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0);
1324 default: return Register();
1325}
1326}
1327
1328Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Register Op0) {
1329 if ((!Subtarget->isLittleEndian())) {
1330 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1331 }
1332 return Register();
1333}
1334
1335Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Register Op0) {
1336 if ((!Subtarget->isLittleEndian())) {
1337 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1338 }
1339 return Register();
1340}
1341
1342Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Register Op0) {
1343 if ((!Subtarget->isLittleEndian())) {
1344 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1345 }
1346 return Register();
1347}
1348
1349Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Register Op0) {
1350 if ((!Subtarget->isLittleEndian())) {
1351 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1352 }
1353 return Register();
1354}
1355
1356Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Register Op0) {
1357 if ((!Subtarget->isLittleEndian())) {
1358 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1359 }
1360 return Register();
1361}
1362
1363Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Register Op0) {
1364 if ((!Subtarget->isLittleEndian())) {
1365 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1366 }
1367 return Register();
1368}
1369
1370Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Register Op0) {
1371 if ((!Subtarget->isLittleEndian())) {
1372 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1373 }
1374 return Register();
1375}
1376
1377Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Register Op0) {
1378 if ((!Subtarget->isLittleEndian())) {
1379 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1380 }
1381 return Register();
1382}
1383
1384Register fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, Register Op0) {
1385switch (RetVT.SimpleTy) {
1386 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0);
1387 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0);
1388 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0);
1389 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0);
1390 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0);
1391 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Op0);
1392 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0);
1393 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Op0);
1394 default: return Register();
1395}
1396}
1397
1398Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Register Op0) {
1399 if ((!Subtarget->isLittleEndian())) {
1400 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1401 }
1402 return Register();
1403}
1404
1405Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Register Op0) {
1406 if ((!Subtarget->isLittleEndian())) {
1407 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1408 }
1409 return Register();
1410}
1411
1412Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Register Op0) {
1413 if ((!Subtarget->isLittleEndian())) {
1414 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1415 }
1416 return Register();
1417}
1418
1419Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Register Op0) {
1420 if ((!Subtarget->isLittleEndian())) {
1421 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1422 }
1423 return Register();
1424}
1425
1426Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Register Op0) {
1427 if ((!Subtarget->isLittleEndian())) {
1428 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1429 }
1430 return Register();
1431}
1432
1433Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Register Op0) {
1434 if ((!Subtarget->isLittleEndian())) {
1435 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1436 }
1437 return Register();
1438}
1439
1440Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Register Op0) {
1441 if ((!Subtarget->isLittleEndian())) {
1442 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1443 }
1444 return Register();
1445}
1446
1447Register fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, Register Op0) {
1448switch (RetVT.SimpleTy) {
1449 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0);
1450 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0);
1451 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0);
1452 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0);
1453 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Op0);
1454 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0);
1455 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0);
1456 default: return Register();
1457}
1458}
1459
1460Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Register Op0) {
1461 if ((!Subtarget->isLittleEndian())) {
1462 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1463 }
1464 return Register();
1465}
1466
1467Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Register Op0) {
1468 if ((!Subtarget->isLittleEndian())) {
1469 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1470 }
1471 return Register();
1472}
1473
1474Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Register Op0) {
1475 if ((!Subtarget->isLittleEndian())) {
1476 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1477 }
1478 return Register();
1479}
1480
1481Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Register Op0) {
1482 if ((!Subtarget->isLittleEndian())) {
1483 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1484 }
1485 return Register();
1486}
1487
1488Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Register Op0) {
1489 if ((!Subtarget->isLittleEndian())) {
1490 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1491 }
1492 return Register();
1493}
1494
1495Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Register Op0) {
1496 if ((!Subtarget->isLittleEndian())) {
1497 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1498 }
1499 return Register();
1500}
1501
1502Register fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, Register Op0) {
1503switch (RetVT.SimpleTy) {
1504 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0);
1505 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0);
1506 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0);
1507 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0);
1508 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0);
1509 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Op0);
1510 default: return Register();
1511}
1512}
1513
1514Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Register Op0) {
1515 if ((!Subtarget->isLittleEndian())) {
1516 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1517 }
1518 return Register();
1519}
1520
1521Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Register Op0) {
1522 if ((!Subtarget->isLittleEndian())) {
1523 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1524 }
1525 return Register();
1526}
1527
1528Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Register Op0) {
1529 if ((!Subtarget->isLittleEndian())) {
1530 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1531 }
1532 return Register();
1533}
1534
1535Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Register Op0) {
1536 if ((!Subtarget->isLittleEndian())) {
1537 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1538 }
1539 return Register();
1540}
1541
1542Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Register Op0) {
1543 if ((!Subtarget->isLittleEndian())) {
1544 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1545 }
1546 return Register();
1547}
1548
1549Register fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, Register Op0) {
1550switch (RetVT.SimpleTy) {
1551 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0);
1552 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0);
1553 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0);
1554 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0);
1555 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0);
1556 default: return Register();
1557}
1558}
1559
1560Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Register Op0) {
1561 if ((!Subtarget->isLittleEndian())) {
1562 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1563 }
1564 return Register();
1565}
1566
1567Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Register Op0) {
1568 if ((!Subtarget->isLittleEndian())) {
1569 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1570 }
1571 return Register();
1572}
1573
1574Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Register Op0) {
1575 if ((!Subtarget->isLittleEndian())) {
1576 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1577 }
1578 return Register();
1579}
1580
1581Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Register Op0) {
1582 if ((!Subtarget->isLittleEndian())) {
1583 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1584 }
1585 return Register();
1586}
1587
1588Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Register Op0) {
1589 if ((!Subtarget->isLittleEndian())) {
1590 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1591 }
1592 return Register();
1593}
1594
1595Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Register Op0) {
1596 if ((!Subtarget->isLittleEndian())) {
1597 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1598 }
1599 return Register();
1600}
1601
1602Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Register Op0) {
1603 if ((!Subtarget->isLittleEndian())) {
1604 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1605 }
1606 return Register();
1607}
1608
1609Register fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, Register Op0) {
1610switch (RetVT.SimpleTy) {
1611 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0);
1612 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0);
1613 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0);
1614 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0);
1615 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0);
1616 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Op0);
1617 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Op0);
1618 default: return Register();
1619}
1620}
1621
1622Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Register Op0) {
1623 if ((!Subtarget->isLittleEndian())) {
1624 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1625 }
1626 return Register();
1627}
1628
1629Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Register Op0) {
1630 if ((!Subtarget->isLittleEndian())) {
1631 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1632 }
1633 return Register();
1634}
1635
1636Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Register Op0) {
1637 if ((!Subtarget->isLittleEndian())) {
1638 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1639 }
1640 return Register();
1641}
1642
1643Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Register Op0) {
1644 if ((!Subtarget->isLittleEndian())) {
1645 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1646 }
1647 return Register();
1648}
1649
1650Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Register Op0) {
1651 if ((!Subtarget->isLittleEndian())) {
1652 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1653 }
1654 return Register();
1655}
1656
1657Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Register Op0) {
1658 if ((!Subtarget->isLittleEndian())) {
1659 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1660 }
1661 return Register();
1662}
1663
1664Register fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, Register Op0) {
1665switch (RetVT.SimpleTy) {
1666 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0);
1667 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0);
1668 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0);
1669 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0);
1670 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Op0);
1671 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0);
1672 default: return Register();
1673}
1674}
1675
1676Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Register Op0) {
1677 if ((!Subtarget->isLittleEndian())) {
1678 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1679 }
1680 return Register();
1681}
1682
1683Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Register Op0) {
1684 if ((!Subtarget->isLittleEndian())) {
1685 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1686 }
1687 return Register();
1688}
1689
1690Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Register Op0) {
1691 if ((!Subtarget->isLittleEndian())) {
1692 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1693 }
1694 return Register();
1695}
1696
1697Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Register Op0) {
1698 if ((!Subtarget->isLittleEndian())) {
1699 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1700 }
1701 return Register();
1702}
1703
1704Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Register Op0) {
1705 if ((!Subtarget->isLittleEndian())) {
1706 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1707 }
1708 return Register();
1709}
1710
1711Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Register Op0) {
1712 if ((!Subtarget->isLittleEndian())) {
1713 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1714 }
1715 return Register();
1716}
1717
1718Register fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, Register Op0) {
1719switch (RetVT.SimpleTy) {
1720 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0);
1721 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0);
1722 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0);
1723 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0);
1724 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Op0);
1725 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0);
1726 default: return Register();
1727}
1728}
1729
1730Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Register Op0) {
1731 if ((!Subtarget->isLittleEndian())) {
1732 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1733 }
1734 return Register();
1735}
1736
1737Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Register Op0) {
1738 if ((!Subtarget->isLittleEndian())) {
1739 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1740 }
1741 return Register();
1742}
1743
1744Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Register Op0) {
1745 if ((!Subtarget->isLittleEndian())) {
1746 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1747 }
1748 return Register();
1749}
1750
1751Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Register Op0) {
1752 if ((!Subtarget->isLittleEndian())) {
1753 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1754 }
1755 return Register();
1756}
1757
1758Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Register Op0) {
1759 if ((!Subtarget->isLittleEndian())) {
1760 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1761 }
1762 return Register();
1763}
1764
1765Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Register Op0) {
1766 if ((!Subtarget->isLittleEndian())) {
1767 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1768 }
1769 return Register();
1770}
1771
1772Register fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, Register Op0) {
1773switch (RetVT.SimpleTy) {
1774 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0);
1775 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0);
1776 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0);
1777 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0);
1778 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Op0);
1779 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0);
1780 default: return Register();
1781}
1782}
1783
1784Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Register Op0) {
1785 if ((!Subtarget->isLittleEndian())) {
1786 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1787 }
1788 return Register();
1789}
1790
1791Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Register Op0) {
1792 if ((!Subtarget->isLittleEndian())) {
1793 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1794 }
1795 return Register();
1796}
1797
1798Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Register Op0) {
1799 if ((!Subtarget->isLittleEndian())) {
1800 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1801 }
1802 return Register();
1803}
1804
1805Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Register Op0) {
1806 if ((!Subtarget->isLittleEndian())) {
1807 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1808 }
1809 return Register();
1810}
1811
1812Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Register Op0) {
1813 if ((!Subtarget->isLittleEndian())) {
1814 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1815 }
1816 return Register();
1817}
1818
1819Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Register Op0) {
1820 if ((!Subtarget->isLittleEndian())) {
1821 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1822 }
1823 return Register();
1824}
1825
1826Register fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, Register Op0) {
1827switch (RetVT.SimpleTy) {
1828 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0);
1829 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0);
1830 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0);
1831 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0);
1832 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0);
1833 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Op0);
1834 default: return Register();
1835}
1836}
1837
1838Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Register Op0) {
1839 if ((!Subtarget->isLittleEndian())) {
1840 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1841 }
1842 return Register();
1843}
1844
1845Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Register Op0) {
1846 if ((!Subtarget->isLittleEndian())) {
1847 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1848 }
1849 return Register();
1850}
1851
1852Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Register Op0) {
1853 if ((!Subtarget->isLittleEndian())) {
1854 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1855 }
1856 return Register();
1857}
1858
1859Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Register Op0) {
1860 if ((!Subtarget->isLittleEndian())) {
1861 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1862 }
1863 return Register();
1864}
1865
1866Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Register Op0) {
1867 if ((!Subtarget->isLittleEndian())) {
1868 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1869 }
1870 return Register();
1871}
1872
1873Register fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, Register Op0) {
1874switch (RetVT.SimpleTy) {
1875 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0);
1876 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0);
1877 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0);
1878 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0);
1879 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0);
1880 default: return Register();
1881}
1882}
1883
1884Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Register Op0) {
1885 if ((!Subtarget->isLittleEndian())) {
1886 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1887 }
1888 return Register();
1889}
1890
1891Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Register Op0) {
1892 if ((!Subtarget->isLittleEndian())) {
1893 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1894 }
1895 return Register();
1896}
1897
1898Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Register Op0) {
1899 if ((!Subtarget->isLittleEndian())) {
1900 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1901 }
1902 return Register();
1903}
1904
1905Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Register Op0) {
1906 if ((!Subtarget->isLittleEndian())) {
1907 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1908 }
1909 return Register();
1910}
1911
1912Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Register Op0) {
1913 if ((!Subtarget->isLittleEndian())) {
1914 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1915 }
1916 return Register();
1917}
1918
1919Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(Register Op0) {
1920 if ((!Subtarget->isLittleEndian())) {
1921 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1922 }
1923 return Register();
1924}
1925
1926Register fastEmit_ISD_BITCAST_MVT_v4bf16_r(MVT RetVT, Register Op0) {
1927switch (RetVT.SimpleTy) {
1928 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Op0);
1929 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Op0);
1930 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Op0);
1931 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Op0);
1932 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Op0);
1933 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(Op0);
1934 default: return Register();
1935}
1936}
1937
1938Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Register Op0) {
1939 if ((!Subtarget->isLittleEndian())) {
1940 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1941 }
1942 return Register();
1943}
1944
1945Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Register Op0) {
1946 if ((!Subtarget->isLittleEndian())) {
1947 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1948 }
1949 return Register();
1950}
1951
1952Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Register Op0) {
1953 if ((!Subtarget->isLittleEndian())) {
1954 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1955 }
1956 return Register();
1957}
1958
1959Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Register Op0) {
1960 if ((!Subtarget->isLittleEndian())) {
1961 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1962 }
1963 return Register();
1964}
1965
1966Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Register Op0) {
1967 if ((!Subtarget->isLittleEndian())) {
1968 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1969 }
1970 return Register();
1971}
1972
1973Register fastEmit_ISD_BITCAST_MVT_v8bf16_r(MVT RetVT, Register Op0) {
1974switch (RetVT.SimpleTy) {
1975 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Op0);
1976 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Op0);
1977 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Op0);
1978 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Op0);
1979 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Op0);
1980 default: return Register();
1981}
1982}
1983
1984Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Register Op0) {
1985 if ((!Subtarget->isLittleEndian())) {
1986 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1987 }
1988 return Register();
1989}
1990
1991Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Register Op0) {
1992 if ((!Subtarget->isLittleEndian())) {
1993 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1994 }
1995 return Register();
1996}
1997
1998Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Register Op0) {
1999 if ((!Subtarget->isLittleEndian())) {
2000 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2001 }
2002 return Register();
2003}
2004
2005Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Register Op0) {
2006 if ((!Subtarget->isLittleEndian())) {
2007 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2008 }
2009 return Register();
2010}
2011
2012Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Register Op0) {
2013 if ((!Subtarget->isLittleEndian())) {
2014 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2015 }
2016 return Register();
2017}
2018
2019Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Register Op0) {
2020 if ((!Subtarget->isLittleEndian())) {
2021 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2022 }
2023 return Register();
2024}
2025
2026Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Register Op0) {
2027 if ((!Subtarget->isLittleEndian())) {
2028 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2029 }
2030 return Register();
2031}
2032
2033Register fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, Register Op0) {
2034switch (RetVT.SimpleTy) {
2035 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0);
2036 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0);
2037 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0);
2038 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0);
2039 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0);
2040 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Op0);
2041 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Op0);
2042 default: return Register();
2043}
2044}
2045
2046Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Register Op0) {
2047 if ((!Subtarget->isLittleEndian())) {
2048 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
2049 }
2050 return Register();
2051}
2052
2053Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Register Op0) {
2054 if ((!Subtarget->isLittleEndian())) {
2055 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2056 }
2057 return Register();
2058}
2059
2060Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Register Op0) {
2061 if ((!Subtarget->isLittleEndian())) {
2062 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2063 }
2064 return Register();
2065}
2066
2067Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Register Op0) {
2068 if ((!Subtarget->isLittleEndian())) {
2069 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2070 }
2071 return Register();
2072}
2073
2074Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Register Op0) {
2075 if ((!Subtarget->isLittleEndian())) {
2076 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2077 }
2078 return Register();
2079}
2080
2081Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Register Op0) {
2082 if ((!Subtarget->isLittleEndian())) {
2083 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2084 }
2085 return Register();
2086}
2087
2088Register fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, Register Op0) {
2089switch (RetVT.SimpleTy) {
2090 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0);
2091 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0);
2092 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0);
2093 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0);
2094 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Op0);
2095 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0);
2096 default: return Register();
2097}
2098}
2099
2100Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Register Op0) {
2101 if ((!Subtarget->isLittleEndian())) {
2102 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
2103 }
2104 return Register();
2105}
2106
2107Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Register Op0) {
2108 if ((!Subtarget->isLittleEndian())) {
2109 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2110 }
2111 return Register();
2112}
2113
2114Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Register Op0) {
2115 if ((!Subtarget->isLittleEndian())) {
2116 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2117 }
2118 return Register();
2119}
2120
2121Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Register Op0) {
2122 if ((!Subtarget->isLittleEndian())) {
2123 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2124 }
2125 return Register();
2126}
2127
2128Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(Register Op0) {
2129 if ((!Subtarget->isLittleEndian())) {
2130 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2131 }
2132 return Register();
2133}
2134
2135Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Register Op0) {
2136 if ((!Subtarget->isLittleEndian())) {
2137 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2138 }
2139 return Register();
2140}
2141
2142Register fastEmit_ISD_BITCAST_MVT_v1f64_r(MVT RetVT, Register Op0) {
2143switch (RetVT.SimpleTy) {
2144 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Op0);
2145 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Op0);
2146 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Op0);
2147 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Op0);
2148 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(Op0);
2149 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Op0);
2150 default: return Register();
2151}
2152}
2153
2154Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Register Op0) {
2155 if ((!Subtarget->isLittleEndian())) {
2156 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2157 }
2158 return Register();
2159}
2160
2161Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Register Op0) {
2162 if ((!Subtarget->isLittleEndian())) {
2163 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2164 }
2165 return Register();
2166}
2167
2168Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Register Op0) {
2169 if ((!Subtarget->isLittleEndian())) {
2170 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2171 }
2172 return Register();
2173}
2174
2175Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Register Op0) {
2176 if ((!Subtarget->isLittleEndian())) {
2177 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2178 }
2179 return Register();
2180}
2181
2182Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Register Op0) {
2183 if ((!Subtarget->isLittleEndian())) {
2184 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2185 }
2186 return Register();
2187}
2188
2189Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Register Op0) {
2190 if ((!Subtarget->isLittleEndian())) {
2191 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2192 }
2193 return Register();
2194}
2195
2196Register fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, Register Op0) {
2197switch (RetVT.SimpleTy) {
2198 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0);
2199 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0);
2200 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0);
2201 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0);
2202 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Op0);
2203 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0);
2204 default: return Register();
2205}
2206}
2207
2208Register fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, Register Op0) {
2209 switch (VT.SimpleTy) {
2210 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
2211 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0);
2212 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0);
2213 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0);
2214 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0);
2215 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0);
2216 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0);
2217 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0);
2218 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0);
2219 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0);
2220 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0);
2221 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v4bf16_r(RetVT, Op0);
2222 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v8bf16_r(RetVT, Op0);
2223 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0);
2224 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0);
2225 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v1f64_r(RetVT, Op0);
2226 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0);
2227 default: return Register();
2228 }
2229}
2230
2231// FastEmit functions for ISD::BITREVERSE.
2232
2233Register fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, Register Op0) {
2234 if (RetVT.SimpleTy != MVT::i32)
2235 return Register();
2236 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITWr, RC: &AArch64::GPR32RegClass, Op0);
2237}
2238
2239Register fastEmit_ISD_BITREVERSE_MVT_i64_r(MVT RetVT, Register Op0) {
2240 if (RetVT.SimpleTy != MVT::i64)
2241 return Register();
2242 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITXr, RC: &AArch64::GPR64RegClass, Op0);
2243}
2244
2245Register fastEmit_ISD_BITREVERSE_MVT_v8i8_r(MVT RetVT, Register Op0) {
2246 if (RetVT.SimpleTy != MVT::v8i8)
2247 return Register();
2248 if ((Subtarget->isNeonAvailable())) {
2249 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITv8i8, RC: &AArch64::FPR64RegClass, Op0);
2250 }
2251 return Register();
2252}
2253
2254Register fastEmit_ISD_BITREVERSE_MVT_v16i8_r(MVT RetVT, Register Op0) {
2255 if (RetVT.SimpleTy != MVT::v16i8)
2256 return Register();
2257 if ((Subtarget->isNeonAvailable())) {
2258 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITv16i8, RC: &AArch64::FPR128RegClass, Op0);
2259 }
2260 return Register();
2261}
2262
2263Register fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, Register Op0) {
2264 switch (VT.SimpleTy) {
2265 case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0);
2266 case MVT::i64: return fastEmit_ISD_BITREVERSE_MVT_i64_r(RetVT, Op0);
2267 case MVT::v8i8: return fastEmit_ISD_BITREVERSE_MVT_v8i8_r(RetVT, Op0);
2268 case MVT::v16i8: return fastEmit_ISD_BITREVERSE_MVT_v16i8_r(RetVT, Op0);
2269 default: return Register();
2270 }
2271}
2272
2273// FastEmit functions for ISD::BRIND.
2274
2275Register fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, Register Op0) {
2276 if (RetVT.SimpleTy != MVT::isVoid)
2277 return Register();
2278 return fastEmitInst_r(MachineInstOpcode: AArch64::BR, RC: &AArch64::GPR64RegClass, Op0);
2279}
2280
2281Register fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, Register Op0) {
2282 switch (VT.SimpleTy) {
2283 case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0);
2284 default: return Register();
2285 }
2286}
2287
2288// FastEmit functions for ISD::BSWAP.
2289
2290Register fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, Register Op0) {
2291 if (RetVT.SimpleTy != MVT::i32)
2292 return Register();
2293 return fastEmitInst_r(MachineInstOpcode: AArch64::REVWr, RC: &AArch64::GPR32RegClass, Op0);
2294}
2295
2296Register fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, Register Op0) {
2297 if (RetVT.SimpleTy != MVT::i64)
2298 return Register();
2299 return fastEmitInst_r(MachineInstOpcode: AArch64::REVXr, RC: &AArch64::GPR64RegClass, Op0);
2300}
2301
2302Register fastEmit_ISD_BSWAP_MVT_v4i16_r(MVT RetVT, Register Op0) {
2303 if (RetVT.SimpleTy != MVT::v4i16)
2304 return Register();
2305 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
2306}
2307
2308Register fastEmit_ISD_BSWAP_MVT_v8i16_r(MVT RetVT, Register Op0) {
2309 if (RetVT.SimpleTy != MVT::v8i16)
2310 return Register();
2311 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
2312}
2313
2314Register fastEmit_ISD_BSWAP_MVT_v2i32_r(MVT RetVT, Register Op0) {
2315 if (RetVT.SimpleTy != MVT::v2i32)
2316 return Register();
2317 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
2318}
2319
2320Register fastEmit_ISD_BSWAP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2321 if (RetVT.SimpleTy != MVT::v4i32)
2322 return Register();
2323 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
2324}
2325
2326Register fastEmit_ISD_BSWAP_MVT_v2i64_r(MVT RetVT, Register Op0) {
2327 if (RetVT.SimpleTy != MVT::v2i64)
2328 return Register();
2329 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2330}
2331
2332Register fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, Register Op0) {
2333 switch (VT.SimpleTy) {
2334 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
2335 case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0);
2336 case MVT::v4i16: return fastEmit_ISD_BSWAP_MVT_v4i16_r(RetVT, Op0);
2337 case MVT::v8i16: return fastEmit_ISD_BSWAP_MVT_v8i16_r(RetVT, Op0);
2338 case MVT::v2i32: return fastEmit_ISD_BSWAP_MVT_v2i32_r(RetVT, Op0);
2339 case MVT::v4i32: return fastEmit_ISD_BSWAP_MVT_v4i32_r(RetVT, Op0);
2340 case MVT::v2i64: return fastEmit_ISD_BSWAP_MVT_v2i64_r(RetVT, Op0);
2341 default: return Register();
2342 }
2343}
2344
2345// FastEmit functions for ISD::CTLS.
2346
2347Register fastEmit_ISD_CTLS_MVT_i32_r(MVT RetVT, Register Op0) {
2348 if (RetVT.SimpleTy != MVT::i32)
2349 return Register();
2350 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSWr, RC: &AArch64::GPR32RegClass, Op0);
2351}
2352
2353Register fastEmit_ISD_CTLS_MVT_i64_r(MVT RetVT, Register Op0) {
2354 if (RetVT.SimpleTy != MVT::i64)
2355 return Register();
2356 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSXr, RC: &AArch64::GPR64RegClass, Op0);
2357}
2358
2359Register fastEmit_ISD_CTLS_MVT_v8i8_r(MVT RetVT, Register Op0) {
2360 if (RetVT.SimpleTy != MVT::v8i8)
2361 return Register();
2362 if ((Subtarget->isNeonAvailable())) {
2363 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv8i8, RC: &AArch64::FPR64RegClass, Op0);
2364 }
2365 return Register();
2366}
2367
2368Register fastEmit_ISD_CTLS_MVT_v16i8_r(MVT RetVT, Register Op0) {
2369 if (RetVT.SimpleTy != MVT::v16i8)
2370 return Register();
2371 if ((Subtarget->isNeonAvailable())) {
2372 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv16i8, RC: &AArch64::FPR128RegClass, Op0);
2373 }
2374 return Register();
2375}
2376
2377Register fastEmit_ISD_CTLS_MVT_v4i16_r(MVT RetVT, Register Op0) {
2378 if (RetVT.SimpleTy != MVT::v4i16)
2379 return Register();
2380 if ((Subtarget->isNeonAvailable())) {
2381 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv4i16, RC: &AArch64::FPR64RegClass, Op0);
2382 }
2383 return Register();
2384}
2385
2386Register fastEmit_ISD_CTLS_MVT_v8i16_r(MVT RetVT, Register Op0) {
2387 if (RetVT.SimpleTy != MVT::v8i16)
2388 return Register();
2389 if ((Subtarget->isNeonAvailable())) {
2390 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv8i16, RC: &AArch64::FPR128RegClass, Op0);
2391 }
2392 return Register();
2393}
2394
2395Register fastEmit_ISD_CTLS_MVT_v2i32_r(MVT RetVT, Register Op0) {
2396 if (RetVT.SimpleTy != MVT::v2i32)
2397 return Register();
2398 if ((Subtarget->isNeonAvailable())) {
2399 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv2i32, RC: &AArch64::FPR64RegClass, Op0);
2400 }
2401 return Register();
2402}
2403
2404Register fastEmit_ISD_CTLS_MVT_v4i32_r(MVT RetVT, Register Op0) {
2405 if (RetVT.SimpleTy != MVT::v4i32)
2406 return Register();
2407 if ((Subtarget->isNeonAvailable())) {
2408 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv4i32, RC: &AArch64::FPR128RegClass, Op0);
2409 }
2410 return Register();
2411}
2412
2413Register fastEmit_ISD_CTLS_r(MVT VT, MVT RetVT, Register Op0) {
2414 switch (VT.SimpleTy) {
2415 case MVT::i32: return fastEmit_ISD_CTLS_MVT_i32_r(RetVT, Op0);
2416 case MVT::i64: return fastEmit_ISD_CTLS_MVT_i64_r(RetVT, Op0);
2417 case MVT::v8i8: return fastEmit_ISD_CTLS_MVT_v8i8_r(RetVT, Op0);
2418 case MVT::v16i8: return fastEmit_ISD_CTLS_MVT_v16i8_r(RetVT, Op0);
2419 case MVT::v4i16: return fastEmit_ISD_CTLS_MVT_v4i16_r(RetVT, Op0);
2420 case MVT::v8i16: return fastEmit_ISD_CTLS_MVT_v8i16_r(RetVT, Op0);
2421 case MVT::v2i32: return fastEmit_ISD_CTLS_MVT_v2i32_r(RetVT, Op0);
2422 case MVT::v4i32: return fastEmit_ISD_CTLS_MVT_v4i32_r(RetVT, Op0);
2423 default: return Register();
2424 }
2425}
2426
2427// FastEmit functions for ISD::CTLZ.
2428
2429Register fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, Register Op0) {
2430 if (RetVT.SimpleTy != MVT::i32)
2431 return Register();
2432 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZWr, RC: &AArch64::GPR32RegClass, Op0);
2433}
2434
2435Register fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, Register Op0) {
2436 if (RetVT.SimpleTy != MVT::i64)
2437 return Register();
2438 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZXr, RC: &AArch64::GPR64RegClass, Op0);
2439}
2440
2441Register fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, Register Op0) {
2442 if (RetVT.SimpleTy != MVT::v8i8)
2443 return Register();
2444 if ((Subtarget->isNeonAvailable())) {
2445 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv8i8, RC: &AArch64::FPR64RegClass, Op0);
2446 }
2447 return Register();
2448}
2449
2450Register fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, Register Op0) {
2451 if (RetVT.SimpleTy != MVT::v16i8)
2452 return Register();
2453 if ((Subtarget->isNeonAvailable())) {
2454 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv16i8, RC: &AArch64::FPR128RegClass, Op0);
2455 }
2456 return Register();
2457}
2458
2459Register fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, Register Op0) {
2460 if (RetVT.SimpleTy != MVT::v4i16)
2461 return Register();
2462 if ((Subtarget->isNeonAvailable())) {
2463 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv4i16, RC: &AArch64::FPR64RegClass, Op0);
2464 }
2465 return Register();
2466}
2467
2468Register fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, Register Op0) {
2469 if (RetVT.SimpleTy != MVT::v8i16)
2470 return Register();
2471 if ((Subtarget->isNeonAvailable())) {
2472 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv8i16, RC: &AArch64::FPR128RegClass, Op0);
2473 }
2474 return Register();
2475}
2476
2477Register fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, Register Op0) {
2478 if (RetVT.SimpleTy != MVT::v2i32)
2479 return Register();
2480 if ((Subtarget->isNeonAvailable())) {
2481 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv2i32, RC: &AArch64::FPR64RegClass, Op0);
2482 }
2483 return Register();
2484}
2485
2486Register fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, Register Op0) {
2487 if (RetVT.SimpleTy != MVT::v4i32)
2488 return Register();
2489 if ((Subtarget->isNeonAvailable())) {
2490 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv4i32, RC: &AArch64::FPR128RegClass, Op0);
2491 }
2492 return Register();
2493}
2494
2495Register fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, Register Op0) {
2496 switch (VT.SimpleTy) {
2497 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
2498 case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0);
2499 case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0);
2500 case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0);
2501 case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0);
2502 case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0);
2503 case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0);
2504 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
2505 default: return Register();
2506 }
2507}
2508
2509// FastEmit functions for ISD::CTPOP.
2510
2511Register fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, Register Op0) {
2512 if (RetVT.SimpleTy != MVT::i32)
2513 return Register();
2514 if ((Subtarget->hasCSSC())) {
2515 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTWr, RC: &AArch64::GPR32RegClass, Op0);
2516 }
2517 return Register();
2518}
2519
2520Register fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, Register Op0) {
2521 if (RetVT.SimpleTy != MVT::i64)
2522 return Register();
2523 if ((Subtarget->hasCSSC())) {
2524 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTXr, RC: &AArch64::GPR64RegClass, Op0);
2525 }
2526 return Register();
2527}
2528
2529Register fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, Register Op0) {
2530 if (RetVT.SimpleTy != MVT::v8i8)
2531 return Register();
2532 if ((Subtarget->isNeonAvailable())) {
2533 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTv8i8, RC: &AArch64::FPR64RegClass, Op0);
2534 }
2535 return Register();
2536}
2537
2538Register fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, Register Op0) {
2539 if (RetVT.SimpleTy != MVT::v16i8)
2540 return Register();
2541 if ((Subtarget->isNeonAvailable())) {
2542 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTv16i8, RC: &AArch64::FPR128RegClass, Op0);
2543 }
2544 return Register();
2545}
2546
2547Register fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, Register Op0) {
2548 switch (VT.SimpleTy) {
2549 case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0);
2550 case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0);
2551 case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0);
2552 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
2553 default: return Register();
2554 }
2555}
2556
2557// FastEmit functions for ISD::CTTZ.
2558
2559Register fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, Register Op0) {
2560 if (RetVT.SimpleTy != MVT::i32)
2561 return Register();
2562 if ((Subtarget->hasCSSC())) {
2563 return fastEmitInst_r(MachineInstOpcode: AArch64::CTZWr, RC: &AArch64::GPR32RegClass, Op0);
2564 }
2565 return Register();
2566}
2567
2568Register fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, Register Op0) {
2569 if (RetVT.SimpleTy != MVT::i64)
2570 return Register();
2571 if ((Subtarget->hasCSSC())) {
2572 return fastEmitInst_r(MachineInstOpcode: AArch64::CTZXr, RC: &AArch64::GPR64RegClass, Op0);
2573 }
2574 return Register();
2575}
2576
2577Register fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, Register Op0) {
2578 switch (VT.SimpleTy) {
2579 case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0);
2580 case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0);
2581 default: return Register();
2582 }
2583}
2584
2585// FastEmit functions for ISD::FABS.
2586
2587Register fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, Register Op0) {
2588 if (RetVT.SimpleTy != MVT::f16)
2589 return Register();
2590 if ((Subtarget->hasFullFP16())) {
2591 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSHr, RC: &AArch64::FPR16RegClass, Op0);
2592 }
2593 return Register();
2594}
2595
2596Register fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, Register Op0) {
2597 if (RetVT.SimpleTy != MVT::f32)
2598 return Register();
2599 if ((Subtarget->hasFPARMv8())) {
2600 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSSr, RC: &AArch64::FPR32RegClass, Op0);
2601 }
2602 return Register();
2603}
2604
2605Register fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, Register Op0) {
2606 if (RetVT.SimpleTy != MVT::f64)
2607 return Register();
2608 if ((Subtarget->hasFPARMv8())) {
2609 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSDr, RC: &AArch64::FPR64RegClass, Op0);
2610 }
2611 return Register();
2612}
2613
2614Register fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, Register Op0) {
2615 if (RetVT.SimpleTy != MVT::v4f16)
2616 return Register();
2617 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2618 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f16, RC: &AArch64::FPR64RegClass, Op0);
2619 }
2620 return Register();
2621}
2622
2623Register fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, Register Op0) {
2624 if (RetVT.SimpleTy != MVT::v8f16)
2625 return Register();
2626 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2627 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv8f16, RC: &AArch64::FPR128RegClass, Op0);
2628 }
2629 return Register();
2630}
2631
2632Register fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, Register Op0) {
2633 if (RetVT.SimpleTy != MVT::v2f32)
2634 return Register();
2635 if ((Subtarget->isNeonAvailable())) {
2636 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv2f32, RC: &AArch64::FPR64RegClass, Op0);
2637 }
2638 return Register();
2639}
2640
2641Register fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, Register Op0) {
2642 if (RetVT.SimpleTy != MVT::v4f32)
2643 return Register();
2644 if ((Subtarget->isNeonAvailable())) {
2645 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f32, RC: &AArch64::FPR128RegClass, Op0);
2646 }
2647 return Register();
2648}
2649
2650Register fastEmit_ISD_FABS_MVT_v2f64_r(MVT RetVT, Register Op0) {
2651 if (RetVT.SimpleTy != MVT::v2f64)
2652 return Register();
2653 if ((Subtarget->isNeonAvailable())) {
2654 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv2f64, RC: &AArch64::FPR128RegClass, Op0);
2655 }
2656 return Register();
2657}
2658
2659Register fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, Register Op0) {
2660 switch (VT.SimpleTy) {
2661 case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0);
2662 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
2663 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
2664 case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0);
2665 case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0);
2666 case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0);
2667 case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0);
2668 case MVT::v2f64: return fastEmit_ISD_FABS_MVT_v2f64_r(RetVT, Op0);
2669 default: return Register();
2670 }
2671}
2672
2673// FastEmit functions for ISD::FCEIL.
2674
2675Register fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
2676 if (RetVT.SimpleTy != MVT::f16)
2677 return Register();
2678 if ((Subtarget->hasFullFP16())) {
2679 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPHr, RC: &AArch64::FPR16RegClass, Op0);
2680 }
2681 return Register();
2682}
2683
2684Register fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
2685 if (RetVT.SimpleTy != MVT::f32)
2686 return Register();
2687 if ((Subtarget->hasFPARMv8())) {
2688 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPSr, RC: &AArch64::FPR32RegClass, Op0);
2689 }
2690 return Register();
2691}
2692
2693Register fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
2694 if (RetVT.SimpleTy != MVT::f64)
2695 return Register();
2696 if ((Subtarget->hasFPARMv8())) {
2697 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPDr, RC: &AArch64::FPR64RegClass, Op0);
2698 }
2699 return Register();
2700}
2701
2702Register fastEmit_ISD_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
2703 if (RetVT.SimpleTy != MVT::v4f16)
2704 return Register();
2705 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2706 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f16, RC: &AArch64::FPR64RegClass, Op0);
2707 }
2708 return Register();
2709}
2710
2711Register fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
2712 if (RetVT.SimpleTy != MVT::v8f16)
2713 return Register();
2714 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2715 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv8f16, RC: &AArch64::FPR128RegClass, Op0);
2716 }
2717 return Register();
2718}
2719
2720Register fastEmit_ISD_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
2721 if (RetVT.SimpleTy != MVT::v2f32)
2722 return Register();
2723 if ((Subtarget->isNeonAvailable())) {
2724 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f32, RC: &AArch64::FPR64RegClass, Op0);
2725 }
2726 return Register();
2727}
2728
2729Register fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
2730 if (RetVT.SimpleTy != MVT::v4f32)
2731 return Register();
2732 if ((Subtarget->isNeonAvailable())) {
2733 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f32, RC: &AArch64::FPR128RegClass, Op0);
2734 }
2735 return Register();
2736}
2737
2738Register fastEmit_ISD_FCEIL_MVT_v2f64_r(MVT RetVT, Register Op0) {
2739 if (RetVT.SimpleTy != MVT::v2f64)
2740 return Register();
2741 if ((Subtarget->isNeonAvailable())) {
2742 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f64, RC: &AArch64::FPR128RegClass, Op0);
2743 }
2744 return Register();
2745}
2746
2747Register fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
2748 switch (VT.SimpleTy) {
2749 case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0);
2750 case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0);
2751 case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0);
2752 case MVT::v4f16: return fastEmit_ISD_FCEIL_MVT_v4f16_r(RetVT, Op0);
2753 case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0);
2754 case MVT::v2f32: return fastEmit_ISD_FCEIL_MVT_v2f32_r(RetVT, Op0);
2755 case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0);
2756 case MVT::v2f64: return fastEmit_ISD_FCEIL_MVT_v2f64_r(RetVT, Op0);
2757 default: return Register();
2758 }
2759}
2760
2761// FastEmit functions for ISD::FFLOOR.
2762
2763Register fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
2764 if (RetVT.SimpleTy != MVT::f16)
2765 return Register();
2766 if ((Subtarget->hasFullFP16())) {
2767 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMHr, RC: &AArch64::FPR16RegClass, Op0);
2768 }
2769 return Register();
2770}
2771
2772Register fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
2773 if (RetVT.SimpleTy != MVT::f32)
2774 return Register();
2775 if ((Subtarget->hasFPARMv8())) {
2776 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMSr, RC: &AArch64::FPR32RegClass, Op0);
2777 }
2778 return Register();
2779}
2780
2781Register fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
2782 if (RetVT.SimpleTy != MVT::f64)
2783 return Register();
2784 if ((Subtarget->hasFPARMv8())) {
2785 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMDr, RC: &AArch64::FPR64RegClass, Op0);
2786 }
2787 return Register();
2788}
2789
2790Register fastEmit_ISD_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
2791 if (RetVT.SimpleTy != MVT::v4f16)
2792 return Register();
2793 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2794 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f16, RC: &AArch64::FPR64RegClass, Op0);
2795 }
2796 return Register();
2797}
2798
2799Register fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
2800 if (RetVT.SimpleTy != MVT::v8f16)
2801 return Register();
2802 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2803 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv8f16, RC: &AArch64::FPR128RegClass, Op0);
2804 }
2805 return Register();
2806}
2807
2808Register fastEmit_ISD_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
2809 if (RetVT.SimpleTy != MVT::v2f32)
2810 return Register();
2811 if ((Subtarget->isNeonAvailable())) {
2812 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f32, RC: &AArch64::FPR64RegClass, Op0);
2813 }
2814 return Register();
2815}
2816
2817Register fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
2818 if (RetVT.SimpleTy != MVT::v4f32)
2819 return Register();
2820 if ((Subtarget->isNeonAvailable())) {
2821 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f32, RC: &AArch64::FPR128RegClass, Op0);
2822 }
2823 return Register();
2824}
2825
2826Register fastEmit_ISD_FFLOOR_MVT_v2f64_r(MVT RetVT, Register Op0) {
2827 if (RetVT.SimpleTy != MVT::v2f64)
2828 return Register();
2829 if ((Subtarget->isNeonAvailable())) {
2830 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f64, RC: &AArch64::FPR128RegClass, Op0);
2831 }
2832 return Register();
2833}
2834
2835Register fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
2836 switch (VT.SimpleTy) {
2837 case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0);
2838 case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0);
2839 case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0);
2840 case MVT::v4f16: return fastEmit_ISD_FFLOOR_MVT_v4f16_r(RetVT, Op0);
2841 case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0);
2842 case MVT::v2f32: return fastEmit_ISD_FFLOOR_MVT_v2f32_r(RetVT, Op0);
2843 case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0);
2844 case MVT::v2f64: return fastEmit_ISD_FFLOOR_MVT_v2f64_r(RetVT, Op0);
2845 default: return Register();
2846 }
2847}
2848
2849// FastEmit functions for ISD::FNEARBYINT.
2850
2851Register fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
2852 if (RetVT.SimpleTy != MVT::f16)
2853 return Register();
2854 if ((Subtarget->hasFullFP16())) {
2855 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIHr, RC: &AArch64::FPR16RegClass, Op0);
2856 }
2857 return Register();
2858}
2859
2860Register fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
2861 if (RetVT.SimpleTy != MVT::f32)
2862 return Register();
2863 if ((Subtarget->hasFPARMv8())) {
2864 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTISr, RC: &AArch64::FPR32RegClass, Op0);
2865 }
2866 return Register();
2867}
2868
2869Register fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
2870 if (RetVT.SimpleTy != MVT::f64)
2871 return Register();
2872 if ((Subtarget->hasFPARMv8())) {
2873 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIDr, RC: &AArch64::FPR64RegClass, Op0);
2874 }
2875 return Register();
2876}
2877
2878Register fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2879 if (RetVT.SimpleTy != MVT::v4f16)
2880 return Register();
2881 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2882 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f16, RC: &AArch64::FPR64RegClass, Op0);
2883 }
2884 return Register();
2885}
2886
2887Register fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2888 if (RetVT.SimpleTy != MVT::v8f16)
2889 return Register();
2890 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2891 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv8f16, RC: &AArch64::FPR128RegClass, Op0);
2892 }
2893 return Register();
2894}
2895
2896Register fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2897 if (RetVT.SimpleTy != MVT::v2f32)
2898 return Register();
2899 if ((Subtarget->isNeonAvailable())) {
2900 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f32, RC: &AArch64::FPR64RegClass, Op0);
2901 }
2902 return Register();
2903}
2904
2905Register fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2906 if (RetVT.SimpleTy != MVT::v4f32)
2907 return Register();
2908 if ((Subtarget->isNeonAvailable())) {
2909 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f32, RC: &AArch64::FPR128RegClass, Op0);
2910 }
2911 return Register();
2912}
2913
2914Register fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
2915 if (RetVT.SimpleTy != MVT::v2f64)
2916 return Register();
2917 if ((Subtarget->isNeonAvailable())) {
2918 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f64, RC: &AArch64::FPR128RegClass, Op0);
2919 }
2920 return Register();
2921}
2922
2923Register fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
2924 switch (VT.SimpleTy) {
2925 case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0);
2926 case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0);
2927 case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0);
2928 case MVT::v4f16: return fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(RetVT, Op0);
2929 case MVT::v8f16: return fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(RetVT, Op0);
2930 case MVT::v2f32: return fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(RetVT, Op0);
2931 case MVT::v4f32: return fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(RetVT, Op0);
2932 case MVT::v2f64: return fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(RetVT, Op0);
2933 default: return Register();
2934 }
2935}
2936
2937// FastEmit functions for ISD::FNEG.
2938
2939Register fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, Register Op0) {
2940 if (RetVT.SimpleTy != MVT::f16)
2941 return Register();
2942 if ((Subtarget->hasFullFP16())) {
2943 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGHr, RC: &AArch64::FPR16RegClass, Op0);
2944 }
2945 return Register();
2946}
2947
2948Register fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, Register Op0) {
2949 if (RetVT.SimpleTy != MVT::f32)
2950 return Register();
2951 if ((Subtarget->hasFPARMv8())) {
2952 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGSr, RC: &AArch64::FPR32RegClass, Op0);
2953 }
2954 return Register();
2955}
2956
2957Register fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, Register Op0) {
2958 if (RetVT.SimpleTy != MVT::f64)
2959 return Register();
2960 if ((Subtarget->hasFPARMv8())) {
2961 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGDr, RC: &AArch64::FPR64RegClass, Op0);
2962 }
2963 return Register();
2964}
2965
2966Register fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, Register Op0) {
2967 if (RetVT.SimpleTy != MVT::v4f16)
2968 return Register();
2969 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2970 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f16, RC: &AArch64::FPR64RegClass, Op0);
2971 }
2972 return Register();
2973}
2974
2975Register fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, Register Op0) {
2976 if (RetVT.SimpleTy != MVT::v8f16)
2977 return Register();
2978 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2979 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv8f16, RC: &AArch64::FPR128RegClass, Op0);
2980 }
2981 return Register();
2982}
2983
2984Register fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, Register Op0) {
2985 if (RetVT.SimpleTy != MVT::v2f32)
2986 return Register();
2987 if ((Subtarget->isNeonAvailable())) {
2988 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv2f32, RC: &AArch64::FPR64RegClass, Op0);
2989 }
2990 return Register();
2991}
2992
2993Register fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, Register Op0) {
2994 if (RetVT.SimpleTy != MVT::v4f32)
2995 return Register();
2996 if ((Subtarget->isNeonAvailable())) {
2997 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f32, RC: &AArch64::FPR128RegClass, Op0);
2998 }
2999 return Register();
3000}
3001
3002Register fastEmit_ISD_FNEG_MVT_v2f64_r(MVT RetVT, Register Op0) {
3003 if (RetVT.SimpleTy != MVT::v2f64)
3004 return Register();
3005 if ((Subtarget->isNeonAvailable())) {
3006 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv2f64, RC: &AArch64::FPR128RegClass, Op0);
3007 }
3008 return Register();
3009}
3010
3011Register fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, Register Op0) {
3012 switch (VT.SimpleTy) {
3013 case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0);
3014 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
3015 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
3016 case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0);
3017 case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0);
3018 case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0);
3019 case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0);
3020 case MVT::v2f64: return fastEmit_ISD_FNEG_MVT_v2f64_r(RetVT, Op0);
3021 default: return Register();
3022 }
3023}
3024
3025// FastEmit functions for ISD::FP_EXTEND.
3026
3027Register fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Register Op0) {
3028 if ((Subtarget->hasFPARMv8())) {
3029 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSHr, RC: &AArch64::FPR32RegClass, Op0);
3030 }
3031 return Register();
3032}
3033
3034Register fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Register Op0) {
3035 if ((Subtarget->hasFPARMv8())) {
3036 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDHr, RC: &AArch64::FPR64RegClass, Op0);
3037 }
3038 return Register();
3039}
3040
3041Register fastEmit_ISD_FP_EXTEND_MVT_f16_r(MVT RetVT, Register Op0) {
3042switch (RetVT.SimpleTy) {
3043 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Op0);
3044 case MVT::f64: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Op0);
3045 default: return Register();
3046}
3047}
3048
3049Register fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
3050 if (RetVT.SimpleTy != MVT::f64)
3051 return Register();
3052 if ((Subtarget->hasFPARMv8())) {
3053 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDSr, RC: &AArch64::FPR64RegClass, Op0);
3054 }
3055 return Register();
3056}
3057
3058Register fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3059 if (RetVT.SimpleTy != MVT::v4f32)
3060 return Register();
3061 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv4i16, RC: &AArch64::FPR128RegClass, Op0);
3062}
3063
3064Register fastEmit_ISD_FP_EXTEND_MVT_v4bf16_r(MVT RetVT, Register Op0) {
3065 if (RetVT.SimpleTy != MVT::v4f32)
3066 return Register();
3067 if ((Subtarget->isNeonAvailable())) {
3068 return fastEmitInst_r(MachineInstOpcode: AArch64::SHLLv4i16, RC: &AArch64::FPR128RegClass, Op0);
3069 }
3070 return Register();
3071}
3072
3073Register fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3074 if (RetVT.SimpleTy != MVT::v2f64)
3075 return Register();
3076 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv2i32, RC: &AArch64::FPR128RegClass, Op0);
3077}
3078
3079Register fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3080 switch (VT.SimpleTy) {
3081 case MVT::f16: return fastEmit_ISD_FP_EXTEND_MVT_f16_r(RetVT, Op0);
3082 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
3083 case MVT::v4f16: return fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
3084 case MVT::v4bf16: return fastEmit_ISD_FP_EXTEND_MVT_v4bf16_r(RetVT, Op0);
3085 case MVT::v2f32: return fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(RetVT, Op0);
3086 default: return Register();
3087 }
3088}
3089
3090// FastEmit functions for ISD::FP_ROUND.
3091
3092Register fastEmit_ISD_FP_ROUND_MVT_f32_MVT_bf16_r(Register Op0) {
3093 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
3094 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVT, RC: &AArch64::FPR16RegClass, Op0);
3095 }
3096 return Register();
3097}
3098
3099Register fastEmit_ISD_FP_ROUND_MVT_f32_MVT_f16_r(Register Op0) {
3100 if ((Subtarget->hasFPARMv8())) {
3101 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHSr, RC: &AArch64::FPR16RegClass, Op0);
3102 }
3103 return Register();
3104}
3105
3106Register fastEmit_ISD_FP_ROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3107switch (RetVT.SimpleTy) {
3108 case MVT::bf16: return fastEmit_ISD_FP_ROUND_MVT_f32_MVT_bf16_r(Op0);
3109 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f32_MVT_f16_r(Op0);
3110 default: return Register();
3111}
3112}
3113
3114Register fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Register Op0) {
3115 if ((Subtarget->hasFPARMv8())) {
3116 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHDr, RC: &AArch64::FPR16RegClass, Op0);
3117 }
3118 return Register();
3119}
3120
3121Register fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Register Op0) {
3122 if ((Subtarget->hasFPARMv8())) {
3123 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSDr, RC: &AArch64::FPR32RegClass, Op0);
3124 }
3125 return Register();
3126}
3127
3128Register fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3129switch (RetVT.SimpleTy) {
3130 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Op0);
3131 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Op0);
3132 default: return Register();
3133}
3134}
3135
3136Register fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Register Op0) {
3137 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
3138}
3139
3140Register fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Register Op0) {
3141 if ((Subtarget->hasBF16()) && (Subtarget->isNeonAvailable())) {
3142 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVTN, RC: &AArch64::FPR64RegClass, Op0);
3143 }
3144 return Register();
3145}
3146
3147Register fastEmit_ISD_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3148switch (RetVT.SimpleTy) {
3149 case MVT::v4f16: return fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Op0);
3150 case MVT::v4bf16: return fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Op0);
3151 default: return Register();
3152}
3153}
3154
3155Register fastEmit_ISD_FP_ROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
3156 if (RetVT.SimpleTy != MVT::v2f32)
3157 return Register();
3158 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
3159}
3160
3161Register fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
3162 switch (VT.SimpleTy) {
3163 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f32_r(RetVT, Op0);
3164 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
3165 case MVT::v4f32: return fastEmit_ISD_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
3166 case MVT::v2f64: return fastEmit_ISD_FP_ROUND_MVT_v2f64_r(RetVT, Op0);
3167 default: return Register();
3168 }
3169}
3170
3171// FastEmit functions for ISD::FP_TO_SINT.
3172
3173Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
3174 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
3175 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSHr, RC: &AArch64::FPR32RegClass, Op0);
3176 }
3177 if ((Subtarget->hasFullFP16())) {
3178 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
3179 }
3180 return Register();
3181}
3182
3183Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
3184 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
3185 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDHr, RC: &AArch64::FPR64RegClass, Op0);
3186 }
3187 if ((Subtarget->hasFullFP16())) {
3188 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
3189 }
3190 return Register();
3191}
3192
3193Register fastEmit_ISD_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
3194switch (RetVT.SimpleTy) {
3195 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
3196 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
3197 default: return Register();
3198}
3199}
3200
3201Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
3202 if ((Subtarget->hasFPARMv8())) {
3203 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
3204 }
3205 return Register();
3206}
3207
3208Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
3209 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
3210 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDSr, RC: &AArch64::FPR64RegClass, Op0);
3211 }
3212 if ((Subtarget->hasFPARMv8())) {
3213 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
3214 }
3215 return Register();
3216}
3217
3218Register fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
3219switch (RetVT.SimpleTy) {
3220 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
3221 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
3222 default: return Register();
3223}
3224}
3225
3226Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
3227 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
3228 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSDr, RC: &AArch64::FPR32RegClass, Op0);
3229 }
3230 if ((Subtarget->hasFPARMv8())) {
3231 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
3232 }
3233 return Register();
3234}
3235
3236Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
3237 if ((Subtarget->hasFPARMv8())) {
3238 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
3239 }
3240 return Register();
3241}
3242
3243Register fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
3244switch (RetVT.SimpleTy) {
3245 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
3246 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
3247 default: return Register();
3248}
3249}
3250
3251Register fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3252 if (RetVT.SimpleTy != MVT::v4i16)
3253 return Register();
3254 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3255 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
3256 }
3257 return Register();
3258}
3259
3260Register fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3261 if (RetVT.SimpleTy != MVT::v8i16)
3262 return Register();
3263 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3264 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
3265 }
3266 return Register();
3267}
3268
3269Register fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3270 if (RetVT.SimpleTy != MVT::v2i32)
3271 return Register();
3272 if ((Subtarget->isNeonAvailable())) {
3273 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
3274 }
3275 return Register();
3276}
3277
3278Register fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3279 if (RetVT.SimpleTy != MVT::v4i32)
3280 return Register();
3281 if ((Subtarget->isNeonAvailable())) {
3282 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
3283 }
3284 return Register();
3285}
3286
3287Register fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3288 if (RetVT.SimpleTy != MVT::v2i64)
3289 return Register();
3290 if ((Subtarget->isNeonAvailable())) {
3291 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
3292 }
3293 return Register();
3294}
3295
3296Register fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
3297 switch (VT.SimpleTy) {
3298 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
3299 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
3300 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
3301 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
3302 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
3303 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
3304 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
3305 case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0);
3306 default: return Register();
3307 }
3308}
3309
3310// FastEmit functions for ISD::FP_TO_SINT_SAT.
3311
3312Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_MVT_i32_r(Register Op0) {
3313 if ((Subtarget->hasFPRCVT())) {
3314 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSHr, RC: &AArch64::FPR32RegClass, Op0);
3315 }
3316 if ((Subtarget->hasFullFP16())) {
3317 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
3318 }
3319 return Register();
3320}
3321
3322Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_MVT_i64_r(Register Op0) {
3323 if ((Subtarget->hasFPRCVT())) {
3324 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDHr, RC: &AArch64::FPR64RegClass, Op0);
3325 }
3326 if ((Subtarget->hasFullFP16())) {
3327 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
3328 }
3329 return Register();
3330}
3331
3332Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_r(MVT RetVT, Register Op0) {
3333switch (RetVT.SimpleTy) {
3334 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_MVT_i32_r(Op0);
3335 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_MVT_i64_r(Op0);
3336 default: return Register();
3337}
3338}
3339
3340Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_MVT_i32_r(Register Op0) {
3341 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
3342}
3343
3344Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_MVT_i64_r(Register Op0) {
3345 if ((Subtarget->hasFPRCVT())) {
3346 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDSr, RC: &AArch64::FPR64RegClass, Op0);
3347 }
3348 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
3349}
3350
3351Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_r(MVT RetVT, Register Op0) {
3352switch (RetVT.SimpleTy) {
3353 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_MVT_i32_r(Op0);
3354 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_MVT_i64_r(Op0);
3355 default: return Register();
3356}
3357}
3358
3359Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_MVT_i32_r(Register Op0) {
3360 if ((Subtarget->hasFPRCVT())) {
3361 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSDr, RC: &AArch64::FPR32RegClass, Op0);
3362 }
3363 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
3364}
3365
3366Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_MVT_i64_r(Register Op0) {
3367 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
3368}
3369
3370Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_r(MVT RetVT, Register Op0) {
3371switch (RetVT.SimpleTy) {
3372 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_MVT_i32_r(Op0);
3373 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_MVT_i64_r(Op0);
3374 default: return Register();
3375}
3376}
3377
3378Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3379 if (RetVT.SimpleTy != MVT::v4i16)
3380 return Register();
3381 if ((Subtarget->hasFullFP16())) {
3382 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
3383 }
3384 return Register();
3385}
3386
3387Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3388 if (RetVT.SimpleTy != MVT::v8i16)
3389 return Register();
3390 if ((Subtarget->hasFullFP16())) {
3391 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
3392 }
3393 return Register();
3394}
3395
3396Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3397 if (RetVT.SimpleTy != MVT::v2i32)
3398 return Register();
3399 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
3400}
3401
3402Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3403 if (RetVT.SimpleTy != MVT::v4i32)
3404 return Register();
3405 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
3406}
3407
3408Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3409 if (RetVT.SimpleTy != MVT::v2i64)
3410 return Register();
3411 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
3412}
3413
3414Register fastEmit_ISD_FP_TO_SINT_SAT_r(MVT VT, MVT RetVT, Register Op0) {
3415 switch (VT.SimpleTy) {
3416 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_r(RetVT, Op0);
3417 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_r(RetVT, Op0);
3418 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_r(RetVT, Op0);
3419 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_v4f16_r(RetVT, Op0);
3420 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_v8f16_r(RetVT, Op0);
3421 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_v2f32_r(RetVT, Op0);
3422 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_v4f32_r(RetVT, Op0);
3423 case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_v2f64_r(RetVT, Op0);
3424 default: return Register();
3425 }
3426}
3427
3428// FastEmit functions for ISD::FP_TO_UINT.
3429
3430Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
3431 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
3432 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSHr, RC: &AArch64::FPR32RegClass, Op0);
3433 }
3434 if ((Subtarget->hasFullFP16())) {
3435 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
3436 }
3437 return Register();
3438}
3439
3440Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
3441 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
3442 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDHr, RC: &AArch64::FPR64RegClass, Op0);
3443 }
3444 if ((Subtarget->hasFullFP16())) {
3445 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
3446 }
3447 return Register();
3448}
3449
3450Register fastEmit_ISD_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
3451switch (RetVT.SimpleTy) {
3452 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
3453 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
3454 default: return Register();
3455}
3456}
3457
3458Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
3459 if ((Subtarget->hasFPARMv8())) {
3460 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
3461 }
3462 return Register();
3463}
3464
3465Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
3466 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
3467 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDSr, RC: &AArch64::FPR64RegClass, Op0);
3468 }
3469 if ((Subtarget->hasFPARMv8())) {
3470 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
3471 }
3472 return Register();
3473}
3474
3475Register fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
3476switch (RetVT.SimpleTy) {
3477 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
3478 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
3479 default: return Register();
3480}
3481}
3482
3483Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
3484 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
3485 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSDr, RC: &AArch64::FPR32RegClass, Op0);
3486 }
3487 if ((Subtarget->hasFPARMv8())) {
3488 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
3489 }
3490 return Register();
3491}
3492
3493Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
3494 if ((Subtarget->hasFPARMv8())) {
3495 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
3496 }
3497 return Register();
3498}
3499
3500Register fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
3501switch (RetVT.SimpleTy) {
3502 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
3503 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
3504 default: return Register();
3505}
3506}
3507
3508Register fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3509 if (RetVT.SimpleTy != MVT::v4i16)
3510 return Register();
3511 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3512 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
3513 }
3514 return Register();
3515}
3516
3517Register fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3518 if (RetVT.SimpleTy != MVT::v8i16)
3519 return Register();
3520 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3521 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
3522 }
3523 return Register();
3524}
3525
3526Register fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3527 if (RetVT.SimpleTy != MVT::v2i32)
3528 return Register();
3529 if ((Subtarget->isNeonAvailable())) {
3530 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
3531 }
3532 return Register();
3533}
3534
3535Register fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3536 if (RetVT.SimpleTy != MVT::v4i32)
3537 return Register();
3538 if ((Subtarget->isNeonAvailable())) {
3539 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
3540 }
3541 return Register();
3542}
3543
3544Register fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3545 if (RetVT.SimpleTy != MVT::v2i64)
3546 return Register();
3547 if ((Subtarget->isNeonAvailable())) {
3548 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
3549 }
3550 return Register();
3551}
3552
3553Register fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
3554 switch (VT.SimpleTy) {
3555 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
3556 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
3557 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
3558 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
3559 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
3560 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
3561 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
3562 case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0);
3563 default: return Register();
3564 }
3565}
3566
3567// FastEmit functions for ISD::FP_TO_UINT_SAT.
3568
3569Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_MVT_i32_r(Register Op0) {
3570 if ((Subtarget->hasFPRCVT())) {
3571 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSHr, RC: &AArch64::FPR32RegClass, Op0);
3572 }
3573 if ((Subtarget->hasFullFP16())) {
3574 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
3575 }
3576 return Register();
3577}
3578
3579Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_MVT_i64_r(Register Op0) {
3580 if ((Subtarget->hasFPRCVT())) {
3581 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDHr, RC: &AArch64::FPR64RegClass, Op0);
3582 }
3583 if ((Subtarget->hasFullFP16())) {
3584 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
3585 }
3586 return Register();
3587}
3588
3589Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_r(MVT RetVT, Register Op0) {
3590switch (RetVT.SimpleTy) {
3591 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_MVT_i32_r(Op0);
3592 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_MVT_i64_r(Op0);
3593 default: return Register();
3594}
3595}
3596
3597Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_MVT_i32_r(Register Op0) {
3598 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
3599}
3600
3601Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_MVT_i64_r(Register Op0) {
3602 if ((Subtarget->hasFPRCVT())) {
3603 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDSr, RC: &AArch64::FPR64RegClass, Op0);
3604 }
3605 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
3606}
3607
3608Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_r(MVT RetVT, Register Op0) {
3609switch (RetVT.SimpleTy) {
3610 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_MVT_i32_r(Op0);
3611 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_MVT_i64_r(Op0);
3612 default: return Register();
3613}
3614}
3615
3616Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_MVT_i32_r(Register Op0) {
3617 if ((Subtarget->hasFPRCVT())) {
3618 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSDr, RC: &AArch64::FPR32RegClass, Op0);
3619 }
3620 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
3621}
3622
3623Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_MVT_i64_r(Register Op0) {
3624 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
3625}
3626
3627Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_r(MVT RetVT, Register Op0) {
3628switch (RetVT.SimpleTy) {
3629 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_MVT_i32_r(Op0);
3630 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_MVT_i64_r(Op0);
3631 default: return Register();
3632}
3633}
3634
3635Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3636 if (RetVT.SimpleTy != MVT::v4i16)
3637 return Register();
3638 if ((Subtarget->hasFullFP16())) {
3639 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
3640 }
3641 return Register();
3642}
3643
3644Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3645 if (RetVT.SimpleTy != MVT::v8i16)
3646 return Register();
3647 if ((Subtarget->hasFullFP16())) {
3648 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
3649 }
3650 return Register();
3651}
3652
3653Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3654 if (RetVT.SimpleTy != MVT::v2i32)
3655 return Register();
3656 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
3657}
3658
3659Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3660 if (RetVT.SimpleTy != MVT::v4i32)
3661 return Register();
3662 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
3663}
3664
3665Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3666 if (RetVT.SimpleTy != MVT::v2i64)
3667 return Register();
3668 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
3669}
3670
3671Register fastEmit_ISD_FP_TO_UINT_SAT_r(MVT VT, MVT RetVT, Register Op0) {
3672 switch (VT.SimpleTy) {
3673 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_r(RetVT, Op0);
3674 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_r(RetVT, Op0);
3675 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_r(RetVT, Op0);
3676 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_v4f16_r(RetVT, Op0);
3677 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_v8f16_r(RetVT, Op0);
3678 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_v2f32_r(RetVT, Op0);
3679 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_v4f32_r(RetVT, Op0);
3680 case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_v2f64_r(RetVT, Op0);
3681 default: return Register();
3682 }
3683}
3684
3685// FastEmit functions for ISD::FRINT.
3686
3687Register fastEmit_ISD_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
3688 if (RetVT.SimpleTy != MVT::f16)
3689 return Register();
3690 if ((Subtarget->hasFullFP16())) {
3691 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXHr, RC: &AArch64::FPR16RegClass, Op0);
3692 }
3693 return Register();
3694}
3695
3696Register fastEmit_ISD_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
3697 if (RetVT.SimpleTy != MVT::f32)
3698 return Register();
3699 if ((Subtarget->hasFPARMv8())) {
3700 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXSr, RC: &AArch64::FPR32RegClass, Op0);
3701 }
3702 return Register();
3703}
3704
3705Register fastEmit_ISD_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
3706 if (RetVT.SimpleTy != MVT::f64)
3707 return Register();
3708 if ((Subtarget->hasFPARMv8())) {
3709 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXDr, RC: &AArch64::FPR64RegClass, Op0);
3710 }
3711 return Register();
3712}
3713
3714Register fastEmit_ISD_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3715 if (RetVT.SimpleTy != MVT::v4f16)
3716 return Register();
3717 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3718 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f16, RC: &AArch64::FPR64RegClass, Op0);
3719 }
3720 return Register();
3721}
3722
3723Register fastEmit_ISD_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3724 if (RetVT.SimpleTy != MVT::v8f16)
3725 return Register();
3726 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3727 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv8f16, RC: &AArch64::FPR128RegClass, Op0);
3728 }
3729 return Register();
3730}
3731
3732Register fastEmit_ISD_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3733 if (RetVT.SimpleTy != MVT::v2f32)
3734 return Register();
3735 if ((Subtarget->isNeonAvailable())) {
3736 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f32, RC: &AArch64::FPR64RegClass, Op0);
3737 }
3738 return Register();
3739}
3740
3741Register fastEmit_ISD_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3742 if (RetVT.SimpleTy != MVT::v4f32)
3743 return Register();
3744 if ((Subtarget->isNeonAvailable())) {
3745 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f32, RC: &AArch64::FPR128RegClass, Op0);
3746 }
3747 return Register();
3748}
3749
3750Register fastEmit_ISD_FRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3751 if (RetVT.SimpleTy != MVT::v2f64)
3752 return Register();
3753 if ((Subtarget->isNeonAvailable())) {
3754 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f64, RC: &AArch64::FPR128RegClass, Op0);
3755 }
3756 return Register();
3757}
3758
3759Register fastEmit_ISD_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
3760 switch (VT.SimpleTy) {
3761 case MVT::f16: return fastEmit_ISD_FRINT_MVT_f16_r(RetVT, Op0);
3762 case MVT::f32: return fastEmit_ISD_FRINT_MVT_f32_r(RetVT, Op0);
3763 case MVT::f64: return fastEmit_ISD_FRINT_MVT_f64_r(RetVT, Op0);
3764 case MVT::v4f16: return fastEmit_ISD_FRINT_MVT_v4f16_r(RetVT, Op0);
3765 case MVT::v8f16: return fastEmit_ISD_FRINT_MVT_v8f16_r(RetVT, Op0);
3766 case MVT::v2f32: return fastEmit_ISD_FRINT_MVT_v2f32_r(RetVT, Op0);
3767 case MVT::v4f32: return fastEmit_ISD_FRINT_MVT_v4f32_r(RetVT, Op0);
3768 case MVT::v2f64: return fastEmit_ISD_FRINT_MVT_v2f64_r(RetVT, Op0);
3769 default: return Register();
3770 }
3771}
3772
3773// FastEmit functions for ISD::FROUND.
3774
3775Register fastEmit_ISD_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
3776 if (RetVT.SimpleTy != MVT::f16)
3777 return Register();
3778 if ((Subtarget->hasFullFP16())) {
3779 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAHr, RC: &AArch64::FPR16RegClass, Op0);
3780 }
3781 return Register();
3782}
3783
3784Register fastEmit_ISD_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3785 if (RetVT.SimpleTy != MVT::f32)
3786 return Register();
3787 if ((Subtarget->hasFPARMv8())) {
3788 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTASr, RC: &AArch64::FPR32RegClass, Op0);
3789 }
3790 return Register();
3791}
3792
3793Register fastEmit_ISD_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3794 if (RetVT.SimpleTy != MVT::f64)
3795 return Register();
3796 if ((Subtarget->hasFPARMv8())) {
3797 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTADr, RC: &AArch64::FPR64RegClass, Op0);
3798 }
3799 return Register();
3800}
3801
3802Register fastEmit_ISD_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3803 if (RetVT.SimpleTy != MVT::v4f16)
3804 return Register();
3805 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3806 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f16, RC: &AArch64::FPR64RegClass, Op0);
3807 }
3808 return Register();
3809}
3810
3811Register fastEmit_ISD_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
3812 if (RetVT.SimpleTy != MVT::v8f16)
3813 return Register();
3814 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3815 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv8f16, RC: &AArch64::FPR128RegClass, Op0);
3816 }
3817 return Register();
3818}
3819
3820Register fastEmit_ISD_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3821 if (RetVT.SimpleTy != MVT::v2f32)
3822 return Register();
3823 if ((Subtarget->isNeonAvailable())) {
3824 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f32, RC: &AArch64::FPR64RegClass, Op0);
3825 }
3826 return Register();
3827}
3828
3829Register fastEmit_ISD_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3830 if (RetVT.SimpleTy != MVT::v4f32)
3831 return Register();
3832 if ((Subtarget->isNeonAvailable())) {
3833 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f32, RC: &AArch64::FPR128RegClass, Op0);
3834 }
3835 return Register();
3836}
3837
3838Register fastEmit_ISD_FROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
3839 if (RetVT.SimpleTy != MVT::v2f64)
3840 return Register();
3841 if ((Subtarget->isNeonAvailable())) {
3842 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f64, RC: &AArch64::FPR128RegClass, Op0);
3843 }
3844 return Register();
3845}
3846
3847Register fastEmit_ISD_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
3848 switch (VT.SimpleTy) {
3849 case MVT::f16: return fastEmit_ISD_FROUND_MVT_f16_r(RetVT, Op0);
3850 case MVT::f32: return fastEmit_ISD_FROUND_MVT_f32_r(RetVT, Op0);
3851 case MVT::f64: return fastEmit_ISD_FROUND_MVT_f64_r(RetVT, Op0);
3852 case MVT::v4f16: return fastEmit_ISD_FROUND_MVT_v4f16_r(RetVT, Op0);
3853 case MVT::v8f16: return fastEmit_ISD_FROUND_MVT_v8f16_r(RetVT, Op0);
3854 case MVT::v2f32: return fastEmit_ISD_FROUND_MVT_v2f32_r(RetVT, Op0);
3855 case MVT::v4f32: return fastEmit_ISD_FROUND_MVT_v4f32_r(RetVT, Op0);
3856 case MVT::v2f64: return fastEmit_ISD_FROUND_MVT_v2f64_r(RetVT, Op0);
3857 default: return Register();
3858 }
3859}
3860
3861// FastEmit functions for ISD::FROUNDEVEN.
3862
3863Register fastEmit_ISD_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
3864 if (RetVT.SimpleTy != MVT::f16)
3865 return Register();
3866 if ((Subtarget->hasFullFP16())) {
3867 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNHr, RC: &AArch64::FPR16RegClass, Op0);
3868 }
3869 return Register();
3870}
3871
3872Register fastEmit_ISD_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
3873 if (RetVT.SimpleTy != MVT::f32)
3874 return Register();
3875 if ((Subtarget->hasFPARMv8())) {
3876 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNSr, RC: &AArch64::FPR32RegClass, Op0);
3877 }
3878 return Register();
3879}
3880
3881Register fastEmit_ISD_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
3882 if (RetVT.SimpleTy != MVT::f64)
3883 return Register();
3884 if ((Subtarget->hasFPARMv8())) {
3885 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNDr, RC: &AArch64::FPR64RegClass, Op0);
3886 }
3887 return Register();
3888}
3889
3890Register fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
3891 if (RetVT.SimpleTy != MVT::v4f16)
3892 return Register();
3893 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3894 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f16, RC: &AArch64::FPR64RegClass, Op0);
3895 }
3896 return Register();
3897}
3898
3899Register fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
3900 if (RetVT.SimpleTy != MVT::v8f16)
3901 return Register();
3902 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3903 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv8f16, RC: &AArch64::FPR128RegClass, Op0);
3904 }
3905 return Register();
3906}
3907
3908Register fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
3909 if (RetVT.SimpleTy != MVT::v2f32)
3910 return Register();
3911 if ((Subtarget->isNeonAvailable())) {
3912 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f32, RC: &AArch64::FPR64RegClass, Op0);
3913 }
3914 return Register();
3915}
3916
3917Register fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
3918 if (RetVT.SimpleTy != MVT::v4f32)
3919 return Register();
3920 if ((Subtarget->isNeonAvailable())) {
3921 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f32, RC: &AArch64::FPR128RegClass, Op0);
3922 }
3923 return Register();
3924}
3925
3926Register fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, Register Op0) {
3927 if (RetVT.SimpleTy != MVT::v2f64)
3928 return Register();
3929 if ((Subtarget->isNeonAvailable())) {
3930 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f64, RC: &AArch64::FPR128RegClass, Op0);
3931 }
3932 return Register();
3933}
3934
3935Register fastEmit_ISD_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
3936 switch (VT.SimpleTy) {
3937 case MVT::f16: return fastEmit_ISD_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
3938 case MVT::f32: return fastEmit_ISD_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
3939 case MVT::f64: return fastEmit_ISD_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
3940 case MVT::v4f16: return fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
3941 case MVT::v8f16: return fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
3942 case MVT::v2f32: return fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
3943 case MVT::v4f32: return fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
3944 case MVT::v2f64: return fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0);
3945 default: return Register();
3946 }
3947}
3948
3949// FastEmit functions for ISD::FSQRT.
3950
3951Register fastEmit_ISD_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
3952 if (RetVT.SimpleTy != MVT::f16)
3953 return Register();
3954 if ((Subtarget->hasFullFP16())) {
3955 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTHr, RC: &AArch64::FPR16RegClass, Op0);
3956 }
3957 return Register();
3958}
3959
3960Register fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
3961 if (RetVT.SimpleTy != MVT::f32)
3962 return Register();
3963 if ((Subtarget->hasFPARMv8())) {
3964 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTSr, RC: &AArch64::FPR32RegClass, Op0);
3965 }
3966 return Register();
3967}
3968
3969Register fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
3970 if (RetVT.SimpleTy != MVT::f64)
3971 return Register();
3972 if ((Subtarget->hasFPARMv8())) {
3973 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTDr, RC: &AArch64::FPR64RegClass, Op0);
3974 }
3975 return Register();
3976}
3977
3978Register fastEmit_ISD_FSQRT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3979 if (RetVT.SimpleTy != MVT::v4f16)
3980 return Register();
3981 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3982 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f16, RC: &AArch64::FPR64RegClass, Op0);
3983 }
3984 return Register();
3985}
3986
3987Register fastEmit_ISD_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3988 if (RetVT.SimpleTy != MVT::v8f16)
3989 return Register();
3990 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3991 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv8f16, RC: &AArch64::FPR128RegClass, Op0);
3992 }
3993 return Register();
3994}
3995
3996Register fastEmit_ISD_FSQRT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3997 if (RetVT.SimpleTy != MVT::v2f32)
3998 return Register();
3999 if ((Subtarget->isNeonAvailable())) {
4000 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f32, RC: &AArch64::FPR64RegClass, Op0);
4001 }
4002 return Register();
4003}
4004
4005Register fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4006 if (RetVT.SimpleTy != MVT::v4f32)
4007 return Register();
4008 if ((Subtarget->isNeonAvailable())) {
4009 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f32, RC: &AArch64::FPR128RegClass, Op0);
4010 }
4011 return Register();
4012}
4013
4014Register fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4015 if (RetVT.SimpleTy != MVT::v2f64)
4016 return Register();
4017 if ((Subtarget->isNeonAvailable())) {
4018 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f64, RC: &AArch64::FPR128RegClass, Op0);
4019 }
4020 return Register();
4021}
4022
4023Register fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
4024 switch (VT.SimpleTy) {
4025 case MVT::f16: return fastEmit_ISD_FSQRT_MVT_f16_r(RetVT, Op0);
4026 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
4027 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
4028 case MVT::v4f16: return fastEmit_ISD_FSQRT_MVT_v4f16_r(RetVT, Op0);
4029 case MVT::v8f16: return fastEmit_ISD_FSQRT_MVT_v8f16_r(RetVT, Op0);
4030 case MVT::v2f32: return fastEmit_ISD_FSQRT_MVT_v2f32_r(RetVT, Op0);
4031 case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0);
4032 case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0);
4033 default: return Register();
4034 }
4035}
4036
4037// FastEmit functions for ISD::FTRUNC.
4038
4039Register fastEmit_ISD_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
4040 if (RetVT.SimpleTy != MVT::f16)
4041 return Register();
4042 if ((Subtarget->hasFullFP16())) {
4043 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZHr, RC: &AArch64::FPR16RegClass, Op0);
4044 }
4045 return Register();
4046}
4047
4048Register fastEmit_ISD_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
4049 if (RetVT.SimpleTy != MVT::f32)
4050 return Register();
4051 if ((Subtarget->hasFPARMv8())) {
4052 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZSr, RC: &AArch64::FPR32RegClass, Op0);
4053 }
4054 return Register();
4055}
4056
4057Register fastEmit_ISD_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
4058 if (RetVT.SimpleTy != MVT::f64)
4059 return Register();
4060 if ((Subtarget->hasFPARMv8())) {
4061 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZDr, RC: &AArch64::FPR64RegClass, Op0);
4062 }
4063 return Register();
4064}
4065
4066Register fastEmit_ISD_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
4067 if (RetVT.SimpleTy != MVT::v4f16)
4068 return Register();
4069 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4070 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f16, RC: &AArch64::FPR64RegClass, Op0);
4071 }
4072 return Register();
4073}
4074
4075Register fastEmit_ISD_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
4076 if (RetVT.SimpleTy != MVT::v8f16)
4077 return Register();
4078 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4079 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv8f16, RC: &AArch64::FPR128RegClass, Op0);
4080 }
4081 return Register();
4082}
4083
4084Register fastEmit_ISD_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
4085 if (RetVT.SimpleTy != MVT::v2f32)
4086 return Register();
4087 if ((Subtarget->isNeonAvailable())) {
4088 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f32, RC: &AArch64::FPR64RegClass, Op0);
4089 }
4090 return Register();
4091}
4092
4093Register fastEmit_ISD_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
4094 if (RetVT.SimpleTy != MVT::v4f32)
4095 return Register();
4096 if ((Subtarget->isNeonAvailable())) {
4097 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f32, RC: &AArch64::FPR128RegClass, Op0);
4098 }
4099 return Register();
4100}
4101
4102Register fastEmit_ISD_FTRUNC_MVT_v2f64_r(MVT RetVT, Register Op0) {
4103 if (RetVT.SimpleTy != MVT::v2f64)
4104 return Register();
4105 if ((Subtarget->isNeonAvailable())) {
4106 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f64, RC: &AArch64::FPR128RegClass, Op0);
4107 }
4108 return Register();
4109}
4110
4111Register fastEmit_ISD_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
4112 switch (VT.SimpleTy) {
4113 case MVT::f16: return fastEmit_ISD_FTRUNC_MVT_f16_r(RetVT, Op0);
4114 case MVT::f32: return fastEmit_ISD_FTRUNC_MVT_f32_r(RetVT, Op0);
4115 case MVT::f64: return fastEmit_ISD_FTRUNC_MVT_f64_r(RetVT, Op0);
4116 case MVT::v4f16: return fastEmit_ISD_FTRUNC_MVT_v4f16_r(RetVT, Op0);
4117 case MVT::v8f16: return fastEmit_ISD_FTRUNC_MVT_v8f16_r(RetVT, Op0);
4118 case MVT::v2f32: return fastEmit_ISD_FTRUNC_MVT_v2f32_r(RetVT, Op0);
4119 case MVT::v4f32: return fastEmit_ISD_FTRUNC_MVT_v4f32_r(RetVT, Op0);
4120 case MVT::v2f64: return fastEmit_ISD_FTRUNC_MVT_v2f64_r(RetVT, Op0);
4121 default: return Register();
4122 }
4123}
4124
4125// FastEmit functions for ISD::LLROUND.
4126
4127Register fastEmit_ISD_LLROUND_MVT_f16_r(MVT RetVT, Register Op0) {
4128 if (RetVT.SimpleTy != MVT::i64)
4129 return Register();
4130 if ((Subtarget->hasFPRCVT())) {
4131 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDHr, RC: &AArch64::FPR64RegClass, Op0);
4132 }
4133 if ((Subtarget->hasFullFP16())) {
4134 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
4135 }
4136 return Register();
4137}
4138
4139Register fastEmit_ISD_LLROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4140 if (RetVT.SimpleTy != MVT::i64)
4141 return Register();
4142 if ((Subtarget->hasFPRCVT())) {
4143 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDSr, RC: &AArch64::FPR64RegClass, Op0);
4144 }
4145 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
4146}
4147
4148Register fastEmit_ISD_LLROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4149 if (RetVT.SimpleTy != MVT::i64)
4150 return Register();
4151 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
4152}
4153
4154Register fastEmit_ISD_LLROUND_r(MVT VT, MVT RetVT, Register Op0) {
4155 switch (VT.SimpleTy) {
4156 case MVT::f16: return fastEmit_ISD_LLROUND_MVT_f16_r(RetVT, Op0);
4157 case MVT::f32: return fastEmit_ISD_LLROUND_MVT_f32_r(RetVT, Op0);
4158 case MVT::f64: return fastEmit_ISD_LLROUND_MVT_f64_r(RetVT, Op0);
4159 default: return Register();
4160 }
4161}
4162
4163// FastEmit functions for ISD::LROUND.
4164
4165Register fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(Register Op0) {
4166 if ((Subtarget->hasFPRCVT())) {
4167 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASSHr, RC: &AArch64::FPR32RegClass, Op0);
4168 }
4169 if ((Subtarget->hasFullFP16())) {
4170 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWHr, RC: &AArch64::GPR32RegClass, Op0);
4171 }
4172 return Register();
4173}
4174
4175Register fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(Register Op0) {
4176 if ((Subtarget->hasFPRCVT())) {
4177 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDHr, RC: &AArch64::FPR64RegClass, Op0);
4178 }
4179 if ((Subtarget->hasFullFP16())) {
4180 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
4181 }
4182 return Register();
4183}
4184
4185Register fastEmit_ISD_LROUND_MVT_f16_r(MVT RetVT, Register Op0) {
4186switch (RetVT.SimpleTy) {
4187 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(Op0);
4188 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(Op0);
4189 default: return Register();
4190}
4191}
4192
4193Register fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(Register Op0) {
4194 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWSr, RC: &AArch64::GPR32RegClass, Op0);
4195}
4196
4197Register fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(Register Op0) {
4198 if ((Subtarget->hasFPRCVT())) {
4199 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDSr, RC: &AArch64::FPR64RegClass, Op0);
4200 }
4201 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
4202}
4203
4204Register fastEmit_ISD_LROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4205switch (RetVT.SimpleTy) {
4206 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(Op0);
4207 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(Op0);
4208 default: return Register();
4209}
4210}
4211
4212Register fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(Register Op0) {
4213 if ((Subtarget->hasFPRCVT())) {
4214 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASSDr, RC: &AArch64::FPR32RegClass, Op0);
4215 }
4216 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWDr, RC: &AArch64::GPR32RegClass, Op0);
4217}
4218
4219Register fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(Register Op0) {
4220 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
4221}
4222
4223Register fastEmit_ISD_LROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4224switch (RetVT.SimpleTy) {
4225 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(Op0);
4226 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(Op0);
4227 default: return Register();
4228}
4229}
4230
4231Register fastEmit_ISD_LROUND_r(MVT VT, MVT RetVT, Register Op0) {
4232 switch (VT.SimpleTy) {
4233 case MVT::f16: return fastEmit_ISD_LROUND_MVT_f16_r(RetVT, Op0);
4234 case MVT::f32: return fastEmit_ISD_LROUND_MVT_f32_r(RetVT, Op0);
4235 case MVT::f64: return fastEmit_ISD_LROUND_MVT_f64_r(RetVT, Op0);
4236 default: return Register();
4237 }
4238}
4239
4240// FastEmit functions for ISD::SINT_TO_FP.
4241
4242Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
4243 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
4244 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
4245 }
4246 if ((Subtarget->hasFullFP16())) {
4247 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
4248 }
4249 return Register();
4250}
4251
4252Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
4253 if ((Subtarget->hasFPARMv8())) {
4254 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
4255 }
4256 return Register();
4257}
4258
4259Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
4260 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
4261 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
4262 }
4263 if ((Subtarget->hasFPARMv8())) {
4264 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
4265 }
4266 return Register();
4267}
4268
4269Register fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
4270switch (RetVT.SimpleTy) {
4271 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
4272 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
4273 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
4274 default: return Register();
4275}
4276}
4277
4278Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
4279 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
4280 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
4281 }
4282 if ((Subtarget->hasFullFP16())) {
4283 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
4284 }
4285 return Register();
4286}
4287
4288Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
4289 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
4290 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
4291 }
4292 if ((Subtarget->hasFPARMv8())) {
4293 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
4294 }
4295 return Register();
4296}
4297
4298Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
4299 if ((Subtarget->hasFPARMv8())) {
4300 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
4301 }
4302 return Register();
4303}
4304
4305Register fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
4306switch (RetVT.SimpleTy) {
4307 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
4308 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
4309 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
4310 default: return Register();
4311}
4312}
4313
4314Register fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
4315 if (RetVT.SimpleTy != MVT::v4f16)
4316 return Register();
4317 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4318 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
4319 }
4320 return Register();
4321}
4322
4323Register fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
4324 if (RetVT.SimpleTy != MVT::v8f16)
4325 return Register();
4326 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4327 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
4328 }
4329 return Register();
4330}
4331
4332Register fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
4333 if (RetVT.SimpleTy != MVT::v2f32)
4334 return Register();
4335 if ((Subtarget->isNeonAvailable())) {
4336 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
4337 }
4338 return Register();
4339}
4340
4341Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
4342 if (RetVT.SimpleTy != MVT::v4f32)
4343 return Register();
4344 if ((Subtarget->isNeonAvailable())) {
4345 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
4346 }
4347 return Register();
4348}
4349
4350Register fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
4351 if (RetVT.SimpleTy != MVT::v2f64)
4352 return Register();
4353 if ((Subtarget->isNeonAvailable())) {
4354 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
4355 }
4356 return Register();
4357}
4358
4359Register fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
4360 switch (VT.SimpleTy) {
4361 case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
4362 case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
4363 case MVT::v4i16: return fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
4364 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
4365 case MVT::v2i32: return fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
4366 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
4367 case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
4368 default: return Register();
4369 }
4370}
4371
4372// FastEmit functions for ISD::SPLAT_VECTOR.
4373
4374Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(Register Op0) {
4375 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4376 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_B, RC: &AArch64::ZPRRegClass, Op0);
4377 }
4378 return Register();
4379}
4380
4381Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(Register Op0) {
4382 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4383 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_H, RC: &AArch64::ZPRRegClass, Op0);
4384 }
4385 return Register();
4386}
4387
4388Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(Register Op0) {
4389 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4390 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_S, RC: &AArch64::ZPRRegClass, Op0);
4391 }
4392 return Register();
4393}
4394
4395Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(MVT RetVT, Register Op0) {
4396switch (RetVT.SimpleTy) {
4397 case MVT::nxv16i8: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(Op0);
4398 case MVT::nxv8i16: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(Op0);
4399 case MVT::nxv4i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(Op0);
4400 default: return Register();
4401}
4402}
4403
4404Register fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(MVT RetVT, Register Op0) {
4405 if (RetVT.SimpleTy != MVT::nxv2i64)
4406 return Register();
4407 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4408 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_D, RC: &AArch64::ZPRRegClass, Op0);
4409 }
4410 return Register();
4411}
4412
4413Register fastEmit_ISD_SPLAT_VECTOR_r(MVT VT, MVT RetVT, Register Op0) {
4414 switch (VT.SimpleTy) {
4415 case MVT::i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(RetVT, Op0);
4416 case MVT::i64: return fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(RetVT, Op0);
4417 default: return Register();
4418 }
4419}
4420
4421// FastEmit functions for ISD::STRICT_FCEIL.
4422
4423Register fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
4424 if (RetVT.SimpleTy != MVT::f16)
4425 return Register();
4426 if ((Subtarget->hasFullFP16())) {
4427 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPHr, RC: &AArch64::FPR16RegClass, Op0);
4428 }
4429 return Register();
4430}
4431
4432Register fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
4433 if (RetVT.SimpleTy != MVT::f32)
4434 return Register();
4435 if ((Subtarget->hasFPARMv8())) {
4436 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPSr, RC: &AArch64::FPR32RegClass, Op0);
4437 }
4438 return Register();
4439}
4440
4441Register fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
4442 if (RetVT.SimpleTy != MVT::f64)
4443 return Register();
4444 if ((Subtarget->hasFPARMv8())) {
4445 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPDr, RC: &AArch64::FPR64RegClass, Op0);
4446 }
4447 return Register();
4448}
4449
4450Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
4451 if (RetVT.SimpleTy != MVT::v4f16)
4452 return Register();
4453 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4454 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f16, RC: &AArch64::FPR64RegClass, Op0);
4455 }
4456 return Register();
4457}
4458
4459Register fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
4460 if (RetVT.SimpleTy != MVT::v8f16)
4461 return Register();
4462 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4463 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv8f16, RC: &AArch64::FPR128RegClass, Op0);
4464 }
4465 return Register();
4466}
4467
4468Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
4469 if (RetVT.SimpleTy != MVT::v2f32)
4470 return Register();
4471 if ((Subtarget->isNeonAvailable())) {
4472 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f32, RC: &AArch64::FPR64RegClass, Op0);
4473 }
4474 return Register();
4475}
4476
4477Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
4478 if (RetVT.SimpleTy != MVT::v4f32)
4479 return Register();
4480 if ((Subtarget->isNeonAvailable())) {
4481 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f32, RC: &AArch64::FPR128RegClass, Op0);
4482 }
4483 return Register();
4484}
4485
4486Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(MVT RetVT, Register Op0) {
4487 if (RetVT.SimpleTy != MVT::v2f64)
4488 return Register();
4489 if ((Subtarget->isNeonAvailable())) {
4490 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f64, RC: &AArch64::FPR128RegClass, Op0);
4491 }
4492 return Register();
4493}
4494
4495Register fastEmit_ISD_STRICT_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
4496 switch (VT.SimpleTy) {
4497 case MVT::f16: return fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(RetVT, Op0);
4498 case MVT::f32: return fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(RetVT, Op0);
4499 case MVT::f64: return fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(RetVT, Op0);
4500 case MVT::v4f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(RetVT, Op0);
4501 case MVT::v8f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(RetVT, Op0);
4502 case MVT::v2f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(RetVT, Op0);
4503 case MVT::v4f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(RetVT, Op0);
4504 case MVT::v2f64: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(RetVT, Op0);
4505 default: return Register();
4506 }
4507}
4508
4509// FastEmit functions for ISD::STRICT_FFLOOR.
4510
4511Register fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
4512 if (RetVT.SimpleTy != MVT::f16)
4513 return Register();
4514 if ((Subtarget->hasFullFP16())) {
4515 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMHr, RC: &AArch64::FPR16RegClass, Op0);
4516 }
4517 return Register();
4518}
4519
4520Register fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
4521 if (RetVT.SimpleTy != MVT::f32)
4522 return Register();
4523 if ((Subtarget->hasFPARMv8())) {
4524 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMSr, RC: &AArch64::FPR32RegClass, Op0);
4525 }
4526 return Register();
4527}
4528
4529Register fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
4530 if (RetVT.SimpleTy != MVT::f64)
4531 return Register();
4532 if ((Subtarget->hasFPARMv8())) {
4533 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMDr, RC: &AArch64::FPR64RegClass, Op0);
4534 }
4535 return Register();
4536}
4537
4538Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
4539 if (RetVT.SimpleTy != MVT::v4f16)
4540 return Register();
4541 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4542 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f16, RC: &AArch64::FPR64RegClass, Op0);
4543 }
4544 return Register();
4545}
4546
4547Register fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
4548 if (RetVT.SimpleTy != MVT::v8f16)
4549 return Register();
4550 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4551 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv8f16, RC: &AArch64::FPR128RegClass, Op0);
4552 }
4553 return Register();
4554}
4555
4556Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
4557 if (RetVT.SimpleTy != MVT::v2f32)
4558 return Register();
4559 if ((Subtarget->isNeonAvailable())) {
4560 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f32, RC: &AArch64::FPR64RegClass, Op0);
4561 }
4562 return Register();
4563}
4564
4565Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
4566 if (RetVT.SimpleTy != MVT::v4f32)
4567 return Register();
4568 if ((Subtarget->isNeonAvailable())) {
4569 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f32, RC: &AArch64::FPR128RegClass, Op0);
4570 }
4571 return Register();
4572}
4573
4574Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(MVT RetVT, Register Op0) {
4575 if (RetVT.SimpleTy != MVT::v2f64)
4576 return Register();
4577 if ((Subtarget->isNeonAvailable())) {
4578 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f64, RC: &AArch64::FPR128RegClass, Op0);
4579 }
4580 return Register();
4581}
4582
4583Register fastEmit_ISD_STRICT_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
4584 switch (VT.SimpleTy) {
4585 case MVT::f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(RetVT, Op0);
4586 case MVT::f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(RetVT, Op0);
4587 case MVT::f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(RetVT, Op0);
4588 case MVT::v4f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(RetVT, Op0);
4589 case MVT::v8f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(RetVT, Op0);
4590 case MVT::v2f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(RetVT, Op0);
4591 case MVT::v4f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(RetVT, Op0);
4592 case MVT::v2f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(RetVT, Op0);
4593 default: return Register();
4594 }
4595}
4596
4597// FastEmit functions for ISD::STRICT_FNEARBYINT.
4598
4599Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
4600 if (RetVT.SimpleTy != MVT::f16)
4601 return Register();
4602 if ((Subtarget->hasFullFP16())) {
4603 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIHr, RC: &AArch64::FPR16RegClass, Op0);
4604 }
4605 return Register();
4606}
4607
4608Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
4609 if (RetVT.SimpleTy != MVT::f32)
4610 return Register();
4611 if ((Subtarget->hasFPARMv8())) {
4612 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTISr, RC: &AArch64::FPR32RegClass, Op0);
4613 }
4614 return Register();
4615}
4616
4617Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
4618 if (RetVT.SimpleTy != MVT::f64)
4619 return Register();
4620 if ((Subtarget->hasFPARMv8())) {
4621 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIDr, RC: &AArch64::FPR64RegClass, Op0);
4622 }
4623 return Register();
4624}
4625
4626Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4627 if (RetVT.SimpleTy != MVT::v4f16)
4628 return Register();
4629 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4630 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f16, RC: &AArch64::FPR64RegClass, Op0);
4631 }
4632 return Register();
4633}
4634
4635Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4636 if (RetVT.SimpleTy != MVT::v8f16)
4637 return Register();
4638 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4639 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv8f16, RC: &AArch64::FPR128RegClass, Op0);
4640 }
4641 return Register();
4642}
4643
4644Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4645 if (RetVT.SimpleTy != MVT::v2f32)
4646 return Register();
4647 if ((Subtarget->isNeonAvailable())) {
4648 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f32, RC: &AArch64::FPR64RegClass, Op0);
4649 }
4650 return Register();
4651}
4652
4653Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4654 if (RetVT.SimpleTy != MVT::v4f32)
4655 return Register();
4656 if ((Subtarget->isNeonAvailable())) {
4657 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f32, RC: &AArch64::FPR128RegClass, Op0);
4658 }
4659 return Register();
4660}
4661
4662Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4663 if (RetVT.SimpleTy != MVT::v2f64)
4664 return Register();
4665 if ((Subtarget->isNeonAvailable())) {
4666 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f64, RC: &AArch64::FPR128RegClass, Op0);
4667 }
4668 return Register();
4669}
4670
4671Register fastEmit_ISD_STRICT_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
4672 switch (VT.SimpleTy) {
4673 case MVT::f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(RetVT, Op0);
4674 case MVT::f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(RetVT, Op0);
4675 case MVT::f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(RetVT, Op0);
4676 case MVT::v4f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(RetVT, Op0);
4677 case MVT::v8f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(RetVT, Op0);
4678 case MVT::v2f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(RetVT, Op0);
4679 case MVT::v4f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(RetVT, Op0);
4680 case MVT::v2f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(RetVT, Op0);
4681 default: return Register();
4682 }
4683}
4684
4685// FastEmit functions for ISD::STRICT_FP_EXTEND.
4686
4687Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(Register Op0) {
4688 if ((Subtarget->hasFPARMv8())) {
4689 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSHr, RC: &AArch64::FPR32RegClass, Op0);
4690 }
4691 return Register();
4692}
4693
4694Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(Register Op0) {
4695 if ((Subtarget->hasFPARMv8())) {
4696 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDHr, RC: &AArch64::FPR64RegClass, Op0);
4697 }
4698 return Register();
4699}
4700
4701Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(MVT RetVT, Register Op0) {
4702switch (RetVT.SimpleTy) {
4703 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(Op0);
4704 case MVT::f64: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(Op0);
4705 default: return Register();
4706}
4707}
4708
4709Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
4710 if (RetVT.SimpleTy != MVT::f64)
4711 return Register();
4712 if ((Subtarget->hasFPARMv8())) {
4713 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDSr, RC: &AArch64::FPR64RegClass, Op0);
4714 }
4715 return Register();
4716}
4717
4718Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
4719 if (RetVT.SimpleTy != MVT::v4f32)
4720 return Register();
4721 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv4i16, RC: &AArch64::FPR128RegClass, Op0);
4722}
4723
4724Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4bf16_r(MVT RetVT, Register Op0) {
4725 if (RetVT.SimpleTy != MVT::v4f32)
4726 return Register();
4727 if ((Subtarget->isNeonAvailable())) {
4728 return fastEmitInst_r(MachineInstOpcode: AArch64::SHLLv4i16, RC: &AArch64::FPR128RegClass, Op0);
4729 }
4730 return Register();
4731}
4732
4733Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(MVT RetVT, Register Op0) {
4734 if (RetVT.SimpleTy != MVT::v2f64)
4735 return Register();
4736 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv2i32, RC: &AArch64::FPR128RegClass, Op0);
4737}
4738
4739Register fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
4740 switch (VT.SimpleTy) {
4741 case MVT::f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(RetVT, Op0);
4742 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0);
4743 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
4744 case MVT::v4bf16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4bf16_r(RetVT, Op0);
4745 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(RetVT, Op0);
4746 default: return Register();
4747 }
4748}
4749
4750// FastEmit functions for ISD::STRICT_FP_ROUND.
4751
4752Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_bf16_r(Register Op0) {
4753 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
4754 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVT, RC: &AArch64::FPR16RegClass, Op0);
4755 }
4756 return Register();
4757}
4758
4759Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_f16_r(Register Op0) {
4760 if ((Subtarget->hasFPARMv8())) {
4761 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHSr, RC: &AArch64::FPR16RegClass, Op0);
4762 }
4763 return Register();
4764}
4765
4766Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4767switch (RetVT.SimpleTy) {
4768 case MVT::bf16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_bf16_r(Op0);
4769 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_f16_r(Op0);
4770 default: return Register();
4771}
4772}
4773
4774Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(Register Op0) {
4775 if ((Subtarget->hasFPARMv8())) {
4776 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHDr, RC: &AArch64::FPR16RegClass, Op0);
4777 }
4778 return Register();
4779}
4780
4781Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(Register Op0) {
4782 if ((Subtarget->hasFPARMv8())) {
4783 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSDr, RC: &AArch64::FPR32RegClass, Op0);
4784 }
4785 return Register();
4786}
4787
4788Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4789switch (RetVT.SimpleTy) {
4790 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(Op0);
4791 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(Op0);
4792 default: return Register();
4793}
4794}
4795
4796Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Register Op0) {
4797 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
4798}
4799
4800Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Register Op0) {
4801 if ((Subtarget->hasBF16()) && (Subtarget->isNeonAvailable())) {
4802 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVTN, RC: &AArch64::FPR64RegClass, Op0);
4803 }
4804 return Register();
4805}
4806
4807Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
4808switch (RetVT.SimpleTy) {
4809 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Op0);
4810 case MVT::v4bf16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Op0);
4811 default: return Register();
4812}
4813}
4814
4815Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
4816 if (RetVT.SimpleTy != MVT::v2f32)
4817 return Register();
4818 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
4819}
4820
4821Register fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
4822 switch (VT.SimpleTy) {
4823 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(RetVT, Op0);
4824 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0);
4825 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
4826 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(RetVT, Op0);
4827 default: return Register();
4828 }
4829}
4830
4831// FastEmit functions for ISD::STRICT_FP_TO_SINT.
4832
4833Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
4834 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
4835 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSHr, RC: &AArch64::FPR32RegClass, Op0);
4836 }
4837 if ((Subtarget->hasFullFP16())) {
4838 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
4839 }
4840 return Register();
4841}
4842
4843Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
4844 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
4845 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDHr, RC: &AArch64::FPR64RegClass, Op0);
4846 }
4847 if ((Subtarget->hasFullFP16())) {
4848 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
4849 }
4850 return Register();
4851}
4852
4853Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
4854switch (RetVT.SimpleTy) {
4855 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
4856 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
4857 default: return Register();
4858}
4859}
4860
4861Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
4862 if ((Subtarget->hasFPARMv8())) {
4863 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
4864 }
4865 return Register();
4866}
4867
4868Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
4869 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
4870 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDSr, RC: &AArch64::FPR64RegClass, Op0);
4871 }
4872 if ((Subtarget->hasFPARMv8())) {
4873 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
4874 }
4875 return Register();
4876}
4877
4878Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
4879switch (RetVT.SimpleTy) {
4880 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
4881 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
4882 default: return Register();
4883}
4884}
4885
4886Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
4887 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
4888 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSDr, RC: &AArch64::FPR32RegClass, Op0);
4889 }
4890 if ((Subtarget->hasFPARMv8())) {
4891 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
4892 }
4893 return Register();
4894}
4895
4896Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
4897 if ((Subtarget->hasFPARMv8())) {
4898 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
4899 }
4900 return Register();
4901}
4902
4903Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
4904switch (RetVT.SimpleTy) {
4905 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
4906 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
4907 default: return Register();
4908}
4909}
4910
4911Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4912 if (RetVT.SimpleTy != MVT::v4i16)
4913 return Register();
4914 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4915 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
4916 }
4917 return Register();
4918}
4919
4920Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4921 if (RetVT.SimpleTy != MVT::v8i16)
4922 return Register();
4923 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4924 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
4925 }
4926 return Register();
4927}
4928
4929Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4930 if (RetVT.SimpleTy != MVT::v2i32)
4931 return Register();
4932 if ((Subtarget->isNeonAvailable())) {
4933 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
4934 }
4935 return Register();
4936}
4937
4938Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4939 if (RetVT.SimpleTy != MVT::v4i32)
4940 return Register();
4941 if ((Subtarget->isNeonAvailable())) {
4942 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
4943 }
4944 return Register();
4945}
4946
4947Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4948 if (RetVT.SimpleTy != MVT::v2i64)
4949 return Register();
4950 if ((Subtarget->isNeonAvailable())) {
4951 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
4952 }
4953 return Register();
4954}
4955
4956Register fastEmit_ISD_STRICT_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
4957 switch (VT.SimpleTy) {
4958 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
4959 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
4960 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
4961 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
4962 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
4963 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
4964 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
4965 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0);
4966 default: return Register();
4967 }
4968}
4969
4970// FastEmit functions for ISD::STRICT_FP_TO_UINT.
4971
4972Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
4973 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
4974 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSHr, RC: &AArch64::FPR32RegClass, Op0);
4975 }
4976 if ((Subtarget->hasFullFP16())) {
4977 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
4978 }
4979 return Register();
4980}
4981
4982Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
4983 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
4984 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDHr, RC: &AArch64::FPR64RegClass, Op0);
4985 }
4986 if ((Subtarget->hasFullFP16())) {
4987 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
4988 }
4989 return Register();
4990}
4991
4992Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
4993switch (RetVT.SimpleTy) {
4994 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
4995 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
4996 default: return Register();
4997}
4998}
4999
5000Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
5001 if ((Subtarget->hasFPARMv8())) {
5002 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
5003 }
5004 return Register();
5005}
5006
5007Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
5008 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
5009 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDSr, RC: &AArch64::FPR64RegClass, Op0);
5010 }
5011 if ((Subtarget->hasFPARMv8())) {
5012 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
5013 }
5014 return Register();
5015}
5016
5017Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
5018switch (RetVT.SimpleTy) {
5019 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
5020 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
5021 default: return Register();
5022}
5023}
5024
5025Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
5026 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
5027 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSDr, RC: &AArch64::FPR32RegClass, Op0);
5028 }
5029 if ((Subtarget->hasFPARMv8())) {
5030 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
5031 }
5032 return Register();
5033}
5034
5035Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
5036 if ((Subtarget->hasFPARMv8())) {
5037 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
5038 }
5039 return Register();
5040}
5041
5042Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
5043switch (RetVT.SimpleTy) {
5044 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
5045 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
5046 default: return Register();
5047}
5048}
5049
5050Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
5051 if (RetVT.SimpleTy != MVT::v4i16)
5052 return Register();
5053 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5054 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
5055 }
5056 return Register();
5057}
5058
5059Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
5060 if (RetVT.SimpleTy != MVT::v8i16)
5061 return Register();
5062 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5063 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
5064 }
5065 return Register();
5066}
5067
5068Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
5069 if (RetVT.SimpleTy != MVT::v2i32)
5070 return Register();
5071 if ((Subtarget->isNeonAvailable())) {
5072 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
5073 }
5074 return Register();
5075}
5076
5077Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
5078 if (RetVT.SimpleTy != MVT::v4i32)
5079 return Register();
5080 if ((Subtarget->isNeonAvailable())) {
5081 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
5082 }
5083 return Register();
5084}
5085
5086Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
5087 if (RetVT.SimpleTy != MVT::v2i64)
5088 return Register();
5089 if ((Subtarget->isNeonAvailable())) {
5090 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
5091 }
5092 return Register();
5093}
5094
5095Register fastEmit_ISD_STRICT_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
5096 switch (VT.SimpleTy) {
5097 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
5098 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
5099 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
5100 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
5101 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
5102 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
5103 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
5104 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0);
5105 default: return Register();
5106 }
5107}
5108
5109// FastEmit functions for ISD::STRICT_FRINT.
5110
5111Register fastEmit_ISD_STRICT_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
5112 if (RetVT.SimpleTy != MVT::f16)
5113 return Register();
5114 if ((Subtarget->hasFullFP16())) {
5115 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXHr, RC: &AArch64::FPR16RegClass, Op0);
5116 }
5117 return Register();
5118}
5119
5120Register fastEmit_ISD_STRICT_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
5121 if (RetVT.SimpleTy != MVT::f32)
5122 return Register();
5123 if ((Subtarget->hasFPARMv8())) {
5124 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXSr, RC: &AArch64::FPR32RegClass, Op0);
5125 }
5126 return Register();
5127}
5128
5129Register fastEmit_ISD_STRICT_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
5130 if (RetVT.SimpleTy != MVT::f64)
5131 return Register();
5132 if ((Subtarget->hasFPARMv8())) {
5133 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXDr, RC: &AArch64::FPR64RegClass, Op0);
5134 }
5135 return Register();
5136}
5137
5138Register fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
5139 if (RetVT.SimpleTy != MVT::v4f16)
5140 return Register();
5141 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5142 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f16, RC: &AArch64::FPR64RegClass, Op0);
5143 }
5144 return Register();
5145}
5146
5147Register fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
5148 if (RetVT.SimpleTy != MVT::v8f16)
5149 return Register();
5150 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5151 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv8f16, RC: &AArch64::FPR128RegClass, Op0);
5152 }
5153 return Register();
5154}
5155
5156Register fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
5157 if (RetVT.SimpleTy != MVT::v2f32)
5158 return Register();
5159 if ((Subtarget->isNeonAvailable())) {
5160 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f32, RC: &AArch64::FPR64RegClass, Op0);
5161 }
5162 return Register();
5163}
5164
5165Register fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
5166 if (RetVT.SimpleTy != MVT::v4f32)
5167 return Register();
5168 if ((Subtarget->isNeonAvailable())) {
5169 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f32, RC: &AArch64::FPR128RegClass, Op0);
5170 }
5171 return Register();
5172}
5173
5174Register fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
5175 if (RetVT.SimpleTy != MVT::v2f64)
5176 return Register();
5177 if ((Subtarget->isNeonAvailable())) {
5178 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f64, RC: &AArch64::FPR128RegClass, Op0);
5179 }
5180 return Register();
5181}
5182
5183Register fastEmit_ISD_STRICT_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
5184 switch (VT.SimpleTy) {
5185 case MVT::f16: return fastEmit_ISD_STRICT_FRINT_MVT_f16_r(RetVT, Op0);
5186 case MVT::f32: return fastEmit_ISD_STRICT_FRINT_MVT_f32_r(RetVT, Op0);
5187 case MVT::f64: return fastEmit_ISD_STRICT_FRINT_MVT_f64_r(RetVT, Op0);
5188 case MVT::v4f16: return fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(RetVT, Op0);
5189 case MVT::v8f16: return fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(RetVT, Op0);
5190 case MVT::v2f32: return fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(RetVT, Op0);
5191 case MVT::v4f32: return fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(RetVT, Op0);
5192 case MVT::v2f64: return fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(RetVT, Op0);
5193 default: return Register();
5194 }
5195}
5196
5197// FastEmit functions for ISD::STRICT_FROUND.
5198
5199Register fastEmit_ISD_STRICT_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5200 if (RetVT.SimpleTy != MVT::f16)
5201 return Register();
5202 if ((Subtarget->hasFullFP16())) {
5203 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAHr, RC: &AArch64::FPR16RegClass, Op0);
5204 }
5205 return Register();
5206}
5207
5208Register fastEmit_ISD_STRICT_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5209 if (RetVT.SimpleTy != MVT::f32)
5210 return Register();
5211 if ((Subtarget->hasFPARMv8())) {
5212 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTASr, RC: &AArch64::FPR32RegClass, Op0);
5213 }
5214 return Register();
5215}
5216
5217Register fastEmit_ISD_STRICT_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5218 if (RetVT.SimpleTy != MVT::f64)
5219 return Register();
5220 if ((Subtarget->hasFPARMv8())) {
5221 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTADr, RC: &AArch64::FPR64RegClass, Op0);
5222 }
5223 return Register();
5224}
5225
5226Register fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
5227 if (RetVT.SimpleTy != MVT::v4f16)
5228 return Register();
5229 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5230 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f16, RC: &AArch64::FPR64RegClass, Op0);
5231 }
5232 return Register();
5233}
5234
5235Register fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
5236 if (RetVT.SimpleTy != MVT::v8f16)
5237 return Register();
5238 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5239 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv8f16, RC: &AArch64::FPR128RegClass, Op0);
5240 }
5241 return Register();
5242}
5243
5244Register fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
5245 if (RetVT.SimpleTy != MVT::v2f32)
5246 return Register();
5247 if ((Subtarget->isNeonAvailable())) {
5248 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f32, RC: &AArch64::FPR64RegClass, Op0);
5249 }
5250 return Register();
5251}
5252
5253Register fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
5254 if (RetVT.SimpleTy != MVT::v4f32)
5255 return Register();
5256 if ((Subtarget->isNeonAvailable())) {
5257 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f32, RC: &AArch64::FPR128RegClass, Op0);
5258 }
5259 return Register();
5260}
5261
5262Register fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
5263 if (RetVT.SimpleTy != MVT::v2f64)
5264 return Register();
5265 if ((Subtarget->isNeonAvailable())) {
5266 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f64, RC: &AArch64::FPR128RegClass, Op0);
5267 }
5268 return Register();
5269}
5270
5271Register fastEmit_ISD_STRICT_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
5272 switch (VT.SimpleTy) {
5273 case MVT::f16: return fastEmit_ISD_STRICT_FROUND_MVT_f16_r(RetVT, Op0);
5274 case MVT::f32: return fastEmit_ISD_STRICT_FROUND_MVT_f32_r(RetVT, Op0);
5275 case MVT::f64: return fastEmit_ISD_STRICT_FROUND_MVT_f64_r(RetVT, Op0);
5276 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(RetVT, Op0);
5277 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(RetVT, Op0);
5278 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(RetVT, Op0);
5279 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(RetVT, Op0);
5280 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(RetVT, Op0);
5281 default: return Register();
5282 }
5283}
5284
5285// FastEmit functions for ISD::STRICT_FROUNDEVEN.
5286
5287Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
5288 if (RetVT.SimpleTy != MVT::f16)
5289 return Register();
5290 if ((Subtarget->hasFullFP16())) {
5291 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNHr, RC: &AArch64::FPR16RegClass, Op0);
5292 }
5293 return Register();
5294}
5295
5296Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
5297 if (RetVT.SimpleTy != MVT::f32)
5298 return Register();
5299 if ((Subtarget->hasFPARMv8())) {
5300 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNSr, RC: &AArch64::FPR32RegClass, Op0);
5301 }
5302 return Register();
5303}
5304
5305Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
5306 if (RetVT.SimpleTy != MVT::f64)
5307 return Register();
5308 if ((Subtarget->hasFPARMv8())) {
5309 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNDr, RC: &AArch64::FPR64RegClass, Op0);
5310 }
5311 return Register();
5312}
5313
5314Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
5315 if (RetVT.SimpleTy != MVT::v4f16)
5316 return Register();
5317 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5318 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f16, RC: &AArch64::FPR64RegClass, Op0);
5319 }
5320 return Register();
5321}
5322
5323Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
5324 if (RetVT.SimpleTy != MVT::v8f16)
5325 return Register();
5326 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5327 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv8f16, RC: &AArch64::FPR128RegClass, Op0);
5328 }
5329 return Register();
5330}
5331
5332Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
5333 if (RetVT.SimpleTy != MVT::v2f32)
5334 return Register();
5335 if ((Subtarget->isNeonAvailable())) {
5336 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f32, RC: &AArch64::FPR64RegClass, Op0);
5337 }
5338 return Register();
5339}
5340
5341Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
5342 if (RetVT.SimpleTy != MVT::v4f32)
5343 return Register();
5344 if ((Subtarget->isNeonAvailable())) {
5345 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f32, RC: &AArch64::FPR128RegClass, Op0);
5346 }
5347 return Register();
5348}
5349
5350Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, Register Op0) {
5351 if (RetVT.SimpleTy != MVT::v2f64)
5352 return Register();
5353 if ((Subtarget->isNeonAvailable())) {
5354 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f64, RC: &AArch64::FPR128RegClass, Op0);
5355 }
5356 return Register();
5357}
5358
5359Register fastEmit_ISD_STRICT_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
5360 switch (VT.SimpleTy) {
5361 case MVT::f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
5362 case MVT::f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
5363 case MVT::f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
5364 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
5365 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
5366 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
5367 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
5368 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0);
5369 default: return Register();
5370 }
5371}
5372
5373// FastEmit functions for ISD::STRICT_FSQRT.
5374
5375Register fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
5376 if (RetVT.SimpleTy != MVT::f16)
5377 return Register();
5378 if ((Subtarget->hasFullFP16())) {
5379 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTHr, RC: &AArch64::FPR16RegClass, Op0);
5380 }
5381 return Register();
5382}
5383
5384Register fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
5385 if (RetVT.SimpleTy != MVT::f32)
5386 return Register();
5387 if ((Subtarget->hasFPARMv8())) {
5388 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTSr, RC: &AArch64::FPR32RegClass, Op0);
5389 }
5390 return Register();
5391}
5392
5393Register fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
5394 if (RetVT.SimpleTy != MVT::f64)
5395 return Register();
5396 if ((Subtarget->hasFPARMv8())) {
5397 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTDr, RC: &AArch64::FPR64RegClass, Op0);
5398 }
5399 return Register();
5400}
5401
5402Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(MVT RetVT, Register Op0) {
5403 if (RetVT.SimpleTy != MVT::v4f16)
5404 return Register();
5405 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5406 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f16, RC: &AArch64::FPR64RegClass, Op0);
5407 }
5408 return Register();
5409}
5410
5411Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
5412 if (RetVT.SimpleTy != MVT::v8f16)
5413 return Register();
5414 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5415 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv8f16, RC: &AArch64::FPR128RegClass, Op0);
5416 }
5417 return Register();
5418}
5419
5420Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(MVT RetVT, Register Op0) {
5421 if (RetVT.SimpleTy != MVT::v2f32)
5422 return Register();
5423 if ((Subtarget->isNeonAvailable())) {
5424 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f32, RC: &AArch64::FPR64RegClass, Op0);
5425 }
5426 return Register();
5427}
5428
5429Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
5430 if (RetVT.SimpleTy != MVT::v4f32)
5431 return Register();
5432 if ((Subtarget->isNeonAvailable())) {
5433 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f32, RC: &AArch64::FPR128RegClass, Op0);
5434 }
5435 return Register();
5436}
5437
5438Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
5439 if (RetVT.SimpleTy != MVT::v2f64)
5440 return Register();
5441 if ((Subtarget->isNeonAvailable())) {
5442 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f64, RC: &AArch64::FPR128RegClass, Op0);
5443 }
5444 return Register();
5445}
5446
5447Register fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
5448 switch (VT.SimpleTy) {
5449 case MVT::f16: return fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(RetVT, Op0);
5450 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0);
5451 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0);
5452 case MVT::v4f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(RetVT, Op0);
5453 case MVT::v8f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(RetVT, Op0);
5454 case MVT::v2f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(RetVT, Op0);
5455 case MVT::v4f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(RetVT, Op0);
5456 case MVT::v2f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(RetVT, Op0);
5457 default: return Register();
5458 }
5459}
5460
5461// FastEmit functions for ISD::STRICT_FTRUNC.
5462
5463Register fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
5464 if (RetVT.SimpleTy != MVT::f16)
5465 return Register();
5466 if ((Subtarget->hasFullFP16())) {
5467 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZHr, RC: &AArch64::FPR16RegClass, Op0);
5468 }
5469 return Register();
5470}
5471
5472Register fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
5473 if (RetVT.SimpleTy != MVT::f32)
5474 return Register();
5475 if ((Subtarget->hasFPARMv8())) {
5476 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZSr, RC: &AArch64::FPR32RegClass, Op0);
5477 }
5478 return Register();
5479}
5480
5481Register fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
5482 if (RetVT.SimpleTy != MVT::f64)
5483 return Register();
5484 if ((Subtarget->hasFPARMv8())) {
5485 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZDr, RC: &AArch64::FPR64RegClass, Op0);
5486 }
5487 return Register();
5488}
5489
5490Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
5491 if (RetVT.SimpleTy != MVT::v4f16)
5492 return Register();
5493 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5494 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f16, RC: &AArch64::FPR64RegClass, Op0);
5495 }
5496 return Register();
5497}
5498
5499Register fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
5500 if (RetVT.SimpleTy != MVT::v8f16)
5501 return Register();
5502 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5503 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv8f16, RC: &AArch64::FPR128RegClass, Op0);
5504 }
5505 return Register();
5506}
5507
5508Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
5509 if (RetVT.SimpleTy != MVT::v2f32)
5510 return Register();
5511 if ((Subtarget->isNeonAvailable())) {
5512 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f32, RC: &AArch64::FPR64RegClass, Op0);
5513 }
5514 return Register();
5515}
5516
5517Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
5518 if (RetVT.SimpleTy != MVT::v4f32)
5519 return Register();
5520 if ((Subtarget->isNeonAvailable())) {
5521 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f32, RC: &AArch64::FPR128RegClass, Op0);
5522 }
5523 return Register();
5524}
5525
5526Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(MVT RetVT, Register Op0) {
5527 if (RetVT.SimpleTy != MVT::v2f64)
5528 return Register();
5529 if ((Subtarget->isNeonAvailable())) {
5530 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f64, RC: &AArch64::FPR128RegClass, Op0);
5531 }
5532 return Register();
5533}
5534
5535Register fastEmit_ISD_STRICT_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
5536 switch (VT.SimpleTy) {
5537 case MVT::f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(RetVT, Op0);
5538 case MVT::f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(RetVT, Op0);
5539 case MVT::f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(RetVT, Op0);
5540 case MVT::v4f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(RetVT, Op0);
5541 case MVT::v8f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(RetVT, Op0);
5542 case MVT::v2f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(RetVT, Op0);
5543 case MVT::v4f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(RetVT, Op0);
5544 case MVT::v2f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(RetVT, Op0);
5545 default: return Register();
5546 }
5547}
5548
5549// FastEmit functions for ISD::STRICT_LLROUND.
5550
5551Register fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5552 if (RetVT.SimpleTy != MVT::i64)
5553 return Register();
5554 if ((Subtarget->hasFullFP16())) {
5555 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
5556 }
5557 return Register();
5558}
5559
5560Register fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5561 if (RetVT.SimpleTy != MVT::i64)
5562 return Register();
5563 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
5564}
5565
5566Register fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5567 if (RetVT.SimpleTy != MVT::i64)
5568 return Register();
5569 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
5570}
5571
5572Register fastEmit_ISD_STRICT_LLROUND_r(MVT VT, MVT RetVT, Register Op0) {
5573 switch (VT.SimpleTy) {
5574 case MVT::f16: return fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(RetVT, Op0);
5575 case MVT::f32: return fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(RetVT, Op0);
5576 case MVT::f64: return fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(RetVT, Op0);
5577 default: return Register();
5578 }
5579}
5580
5581// FastEmit functions for ISD::STRICT_LROUND.
5582
5583Register fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(Register Op0) {
5584 if ((Subtarget->hasFullFP16())) {
5585 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWHr, RC: &AArch64::GPR32RegClass, Op0);
5586 }
5587 return Register();
5588}
5589
5590Register fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(Register Op0) {
5591 if ((Subtarget->hasFullFP16())) {
5592 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
5593 }
5594 return Register();
5595}
5596
5597Register fastEmit_ISD_STRICT_LROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5598switch (RetVT.SimpleTy) {
5599 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(Op0);
5600 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(Op0);
5601 default: return Register();
5602}
5603}
5604
5605Register fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(Register Op0) {
5606 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWSr, RC: &AArch64::GPR32RegClass, Op0);
5607}
5608
5609Register fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(Register Op0) {
5610 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
5611}
5612
5613Register fastEmit_ISD_STRICT_LROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5614switch (RetVT.SimpleTy) {
5615 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(Op0);
5616 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(Op0);
5617 default: return Register();
5618}
5619}
5620
5621Register fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(Register Op0) {
5622 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWDr, RC: &AArch64::GPR32RegClass, Op0);
5623}
5624
5625Register fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(Register Op0) {
5626 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
5627}
5628
5629Register fastEmit_ISD_STRICT_LROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5630switch (RetVT.SimpleTy) {
5631 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(Op0);
5632 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(Op0);
5633 default: return Register();
5634}
5635}
5636
5637Register fastEmit_ISD_STRICT_LROUND_r(MVT VT, MVT RetVT, Register Op0) {
5638 switch (VT.SimpleTy) {
5639 case MVT::f16: return fastEmit_ISD_STRICT_LROUND_MVT_f16_r(RetVT, Op0);
5640 case MVT::f32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_r(RetVT, Op0);
5641 case MVT::f64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_r(RetVT, Op0);
5642 default: return Register();
5643 }
5644}
5645
5646// FastEmit functions for ISD::STRICT_SINT_TO_FP.
5647
5648Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5649 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
5650 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5651 }
5652 if ((Subtarget->hasFullFP16())) {
5653 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5654 }
5655 return Register();
5656}
5657
5658Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5659 if ((Subtarget->hasFPARMv8())) {
5660 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5661 }
5662 return Register();
5663}
5664
5665Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5666 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
5667 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5668 }
5669 if ((Subtarget->hasFPARMv8())) {
5670 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5671 }
5672 return Register();
5673}
5674
5675Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5676switch (RetVT.SimpleTy) {
5677 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5678 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5679 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5680 default: return Register();
5681}
5682}
5683
5684Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5685 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
5686 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5687 }
5688 if ((Subtarget->hasFullFP16())) {
5689 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5690 }
5691 return Register();
5692}
5693
5694Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5695 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
5696 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5697 }
5698 if ((Subtarget->hasFPARMv8())) {
5699 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5700 }
5701 return Register();
5702}
5703
5704Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5705 if ((Subtarget->hasFPARMv8())) {
5706 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5707 }
5708 return Register();
5709}
5710
5711Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5712switch (RetVT.SimpleTy) {
5713 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5714 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5715 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5716 default: return Register();
5717}
5718}
5719
5720Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5721 if (RetVT.SimpleTy != MVT::v4f16)
5722 return Register();
5723 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5724 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5725 }
5726 return Register();
5727}
5728
5729Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5730 if (RetVT.SimpleTy != MVT::v8f16)
5731 return Register();
5732 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5733 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5734 }
5735 return Register();
5736}
5737
5738Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5739 if (RetVT.SimpleTy != MVT::v2f32)
5740 return Register();
5741 if ((Subtarget->isNeonAvailable())) {
5742 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5743 }
5744 return Register();
5745}
5746
5747Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5748 if (RetVT.SimpleTy != MVT::v4f32)
5749 return Register();
5750 if ((Subtarget->isNeonAvailable())) {
5751 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5752 }
5753 return Register();
5754}
5755
5756Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5757 if (RetVT.SimpleTy != MVT::v2f64)
5758 return Register();
5759 if ((Subtarget->isNeonAvailable())) {
5760 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5761 }
5762 return Register();
5763}
5764
5765Register fastEmit_ISD_STRICT_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
5766 switch (VT.SimpleTy) {
5767 case MVT::i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
5768 case MVT::i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
5769 case MVT::v4i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
5770 case MVT::v8i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
5771 case MVT::v2i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
5772 case MVT::v4i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
5773 case MVT::v2i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
5774 default: return Register();
5775 }
5776}
5777
5778// FastEmit functions for ISD::STRICT_UINT_TO_FP.
5779
5780Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5781 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
5782 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5783 }
5784 if ((Subtarget->hasFullFP16())) {
5785 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5786 }
5787 return Register();
5788}
5789
5790Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5791 if ((Subtarget->hasFPARMv8())) {
5792 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5793 }
5794 return Register();
5795}
5796
5797Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5798 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
5799 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5800 }
5801 if ((Subtarget->hasFPARMv8())) {
5802 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5803 }
5804 return Register();
5805}
5806
5807Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5808switch (RetVT.SimpleTy) {
5809 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5810 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5811 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5812 default: return Register();
5813}
5814}
5815
5816Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5817 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
5818 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5819 }
5820 if ((Subtarget->hasFullFP16())) {
5821 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5822 }
5823 return Register();
5824}
5825
5826Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5827 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
5828 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5829 }
5830 if ((Subtarget->hasFPARMv8())) {
5831 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5832 }
5833 return Register();
5834}
5835
5836Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5837 if ((Subtarget->hasFPARMv8())) {
5838 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5839 }
5840 return Register();
5841}
5842
5843Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5844switch (RetVT.SimpleTy) {
5845 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5846 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5847 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5848 default: return Register();
5849}
5850}
5851
5852Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5853 if (RetVT.SimpleTy != MVT::v4f16)
5854 return Register();
5855 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5856 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5857 }
5858 return Register();
5859}
5860
5861Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5862 if (RetVT.SimpleTy != MVT::v8f16)
5863 return Register();
5864 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5865 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5866 }
5867 return Register();
5868}
5869
5870Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5871 if (RetVT.SimpleTy != MVT::v2f32)
5872 return Register();
5873 if ((Subtarget->isNeonAvailable())) {
5874 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5875 }
5876 return Register();
5877}
5878
5879Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5880 if (RetVT.SimpleTy != MVT::v4f32)
5881 return Register();
5882 if ((Subtarget->isNeonAvailable())) {
5883 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5884 }
5885 return Register();
5886}
5887
5888Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5889 if (RetVT.SimpleTy != MVT::v2f64)
5890 return Register();
5891 if ((Subtarget->isNeonAvailable())) {
5892 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5893 }
5894 return Register();
5895}
5896
5897Register fastEmit_ISD_STRICT_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
5898 switch (VT.SimpleTy) {
5899 case MVT::i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(RetVT, Op0);
5900 case MVT::i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(RetVT, Op0);
5901 case MVT::v4i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
5902 case MVT::v8i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
5903 case MVT::v2i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
5904 case MVT::v4i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
5905 case MVT::v2i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
5906 default: return Register();
5907 }
5908}
5909
5910// FastEmit functions for ISD::TRUNCATE.
5911
5912Register fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, Register Op0) {
5913 if (RetVT.SimpleTy != MVT::i32)
5914 return Register();
5915 return fastEmitInst_extractsubreg(RetVT, Op0, Idx: AArch64::sub_32);
5916}
5917
5918Register fastEmit_ISD_TRUNCATE_MVT_v8i16_r(MVT RetVT, Register Op0) {
5919 if (RetVT.SimpleTy != MVT::v8i8)
5920 return Register();
5921 if ((Subtarget->isNeonAvailable())) {
5922 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5923 }
5924 return Register();
5925}
5926
5927Register fastEmit_ISD_TRUNCATE_MVT_v4i32_r(MVT RetVT, Register Op0) {
5928 if (RetVT.SimpleTy != MVT::v4i16)
5929 return Register();
5930 if ((Subtarget->isNeonAvailable())) {
5931 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5932 }
5933 return Register();
5934}
5935
5936Register fastEmit_ISD_TRUNCATE_MVT_v2i64_r(MVT RetVT, Register Op0) {
5937 if (RetVT.SimpleTy != MVT::v2i32)
5938 return Register();
5939 if ((Subtarget->isNeonAvailable())) {
5940 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5941 }
5942 return Register();
5943}
5944
5945Register fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, Register Op0) {
5946 switch (VT.SimpleTy) {
5947 case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0);
5948 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i16_r(RetVT, Op0);
5949 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_MVT_v4i32_r(RetVT, Op0);
5950 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_MVT_v2i64_r(RetVT, Op0);
5951 default: return Register();
5952 }
5953}
5954
5955// FastEmit functions for ISD::TRUNCATE_SSAT_S.
5956
5957Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v8i16_r(MVT RetVT, Register Op0) {
5958 if (RetVT.SimpleTy != MVT::v8i8)
5959 return Register();
5960 if ((Subtarget->isNeonAvailable())) {
5961 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5962 }
5963 return Register();
5964}
5965
5966Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v4i32_r(MVT RetVT, Register Op0) {
5967 if (RetVT.SimpleTy != MVT::v4i16)
5968 return Register();
5969 if ((Subtarget->isNeonAvailable())) {
5970 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5971 }
5972 return Register();
5973}
5974
5975Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v2i64_r(MVT RetVT, Register Op0) {
5976 if (RetVT.SimpleTy != MVT::v2i32)
5977 return Register();
5978 if ((Subtarget->isNeonAvailable())) {
5979 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5980 }
5981 return Register();
5982}
5983
5984Register fastEmit_ISD_TRUNCATE_SSAT_S_r(MVT VT, MVT RetVT, Register Op0) {
5985 switch (VT.SimpleTy) {
5986 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v8i16_r(RetVT, Op0);
5987 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v4i32_r(RetVT, Op0);
5988 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v2i64_r(RetVT, Op0);
5989 default: return Register();
5990 }
5991}
5992
5993// FastEmit functions for ISD::TRUNCATE_SSAT_U.
5994
5995Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v8i16_r(MVT RetVT, Register Op0) {
5996 if (RetVT.SimpleTy != MVT::v8i8)
5997 return Register();
5998 if ((Subtarget->isNeonAvailable())) {
5999 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv8i8, RC: &AArch64::FPR64RegClass, Op0);
6000 }
6001 return Register();
6002}
6003
6004Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v4i32_r(MVT RetVT, Register Op0) {
6005 if (RetVT.SimpleTy != MVT::v4i16)
6006 return Register();
6007 if ((Subtarget->isNeonAvailable())) {
6008 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv4i16, RC: &AArch64::FPR64RegClass, Op0);
6009 }
6010 return Register();
6011}
6012
6013Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v2i64_r(MVT RetVT, Register Op0) {
6014 if (RetVT.SimpleTy != MVT::v2i32)
6015 return Register();
6016 if ((Subtarget->isNeonAvailable())) {
6017 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv2i32, RC: &AArch64::FPR64RegClass, Op0);
6018 }
6019 return Register();
6020}
6021
6022Register fastEmit_ISD_TRUNCATE_SSAT_U_r(MVT VT, MVT RetVT, Register Op0) {
6023 switch (VT.SimpleTy) {
6024 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v8i16_r(RetVT, Op0);
6025 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v4i32_r(RetVT, Op0);
6026 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v2i64_r(RetVT, Op0);
6027 default: return Register();
6028 }
6029}
6030
6031// FastEmit functions for ISD::TRUNCATE_USAT_U.
6032
6033Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v8i16_r(MVT RetVT, Register Op0) {
6034 if (RetVT.SimpleTy != MVT::v8i8)
6035 return Register();
6036 if ((Subtarget->isNeonAvailable())) {
6037 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
6038 }
6039 return Register();
6040}
6041
6042Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v4i32_r(MVT RetVT, Register Op0) {
6043 if (RetVT.SimpleTy != MVT::v4i16)
6044 return Register();
6045 if ((Subtarget->isNeonAvailable())) {
6046 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
6047 }
6048 return Register();
6049}
6050
6051Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v2i64_r(MVT RetVT, Register Op0) {
6052 if (RetVT.SimpleTy != MVT::v2i32)
6053 return Register();
6054 if ((Subtarget->isNeonAvailable())) {
6055 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
6056 }
6057 return Register();
6058}
6059
6060Register fastEmit_ISD_TRUNCATE_USAT_U_r(MVT VT, MVT RetVT, Register Op0) {
6061 switch (VT.SimpleTy) {
6062 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v8i16_r(RetVT, Op0);
6063 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v4i32_r(RetVT, Op0);
6064 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v2i64_r(RetVT, Op0);
6065 default: return Register();
6066 }
6067}
6068
6069// FastEmit functions for ISD::UINT_TO_FP.
6070
6071Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
6072 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
6073 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
6074 }
6075 if ((Subtarget->hasFullFP16())) {
6076 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
6077 }
6078 return Register();
6079}
6080
6081Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
6082 if ((Subtarget->hasFPARMv8())) {
6083 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
6084 }
6085 return Register();
6086}
6087
6088Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
6089 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
6090 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
6091 }
6092 if ((Subtarget->hasFPARMv8())) {
6093 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
6094 }
6095 return Register();
6096}
6097
6098Register fastEmit_ISD_UINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
6099switch (RetVT.SimpleTy) {
6100 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
6101 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
6102 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
6103 default: return Register();
6104}
6105}
6106
6107Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
6108 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
6109 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
6110 }
6111 if ((Subtarget->hasFullFP16())) {
6112 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
6113 }
6114 return Register();
6115}
6116
6117Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
6118 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
6119 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
6120 }
6121 if ((Subtarget->hasFPARMv8())) {
6122 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
6123 }
6124 return Register();
6125}
6126
6127Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
6128 if ((Subtarget->hasFPARMv8())) {
6129 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
6130 }
6131 return Register();
6132}
6133
6134Register fastEmit_ISD_UINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
6135switch (RetVT.SimpleTy) {
6136 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
6137 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
6138 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
6139 default: return Register();
6140}
6141}
6142
6143Register fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
6144 if (RetVT.SimpleTy != MVT::v4f16)
6145 return Register();
6146 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6147 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
6148 }
6149 return Register();
6150}
6151
6152Register fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
6153 if (RetVT.SimpleTy != MVT::v8f16)
6154 return Register();
6155 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6156 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
6157 }
6158 return Register();
6159}
6160
6161Register fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
6162 if (RetVT.SimpleTy != MVT::v2f32)
6163 return Register();
6164 if ((Subtarget->isNeonAvailable())) {
6165 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
6166 }
6167 return Register();
6168}
6169
6170Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
6171 if (RetVT.SimpleTy != MVT::v4f32)
6172 return Register();
6173 if ((Subtarget->isNeonAvailable())) {
6174 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
6175 }
6176 return Register();
6177}
6178
6179Register fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
6180 if (RetVT.SimpleTy != MVT::v2f64)
6181 return Register();
6182 if ((Subtarget->isNeonAvailable())) {
6183 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
6184 }
6185 return Register();
6186}
6187
6188Register fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
6189 switch (VT.SimpleTy) {
6190 case MVT::i32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_r(RetVT, Op0);
6191 case MVT::i64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_r(RetVT, Op0);
6192 case MVT::v4i16: return fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
6193 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
6194 case MVT::v2i32: return fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
6195 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
6196 case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
6197 default: return Register();
6198 }
6199}
6200
6201// FastEmit functions for ISD::VECREDUCE_ADD.
6202
6203Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i8_r(MVT RetVT, Register Op0) {
6204 if (RetVT.SimpleTy != MVT::i8)
6205 return Register();
6206 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6207}
6208
6209Register fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(MVT RetVT, Register Op0) {
6210 if (RetVT.SimpleTy != MVT::i8)
6211 return Register();
6212 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6213}
6214
6215Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i16_r(MVT RetVT, Register Op0) {
6216 if (RetVT.SimpleTy != MVT::i16)
6217 return Register();
6218 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6219}
6220
6221Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(MVT RetVT, Register Op0) {
6222 if (RetVT.SimpleTy != MVT::i16)
6223 return Register();
6224 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6225}
6226
6227Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(MVT RetVT, Register Op0) {
6228 if (RetVT.SimpleTy != MVT::i32)
6229 return Register();
6230 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6231}
6232
6233Register fastEmit_ISD_VECREDUCE_ADD_MVT_v2i64_r(MVT RetVT, Register Op0) {
6234 if (RetVT.SimpleTy != MVT::i64)
6235 return Register();
6236 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6237}
6238
6239Register fastEmit_ISD_VECREDUCE_ADD_r(MVT VT, MVT RetVT, Register Op0) {
6240 switch (VT.SimpleTy) {
6241 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i8_r(RetVT, Op0);
6242 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(RetVT, Op0);
6243 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i16_r(RetVT, Op0);
6244 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(RetVT, Op0);
6245 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(RetVT, Op0);
6246 case MVT::v2i64: return fastEmit_ISD_VECREDUCE_ADD_MVT_v2i64_r(RetVT, Op0);
6247 default: return Register();
6248 }
6249}
6250
6251// FastEmit functions for ISD::VECREDUCE_FADD.
6252
6253Register fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(MVT RetVT, Register Op0) {
6254 if (RetVT.SimpleTy != MVT::f32)
6255 return Register();
6256 return fastEmitInst_r(MachineInstOpcode: AArch64::FADDPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6257}
6258
6259Register fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(MVT RetVT, Register Op0) {
6260 if (RetVT.SimpleTy != MVT::f64)
6261 return Register();
6262 return fastEmitInst_r(MachineInstOpcode: AArch64::FADDPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6263}
6264
6265Register fastEmit_ISD_VECREDUCE_FADD_r(MVT VT, MVT RetVT, Register Op0) {
6266 switch (VT.SimpleTy) {
6267 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(RetVT, Op0);
6268 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(RetVT, Op0);
6269 default: return Register();
6270 }
6271}
6272
6273// FastEmit functions for ISD::VECREDUCE_FMAX.
6274
6275Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f16_r(MVT RetVT, Register Op0) {
6276 if (RetVT.SimpleTy != MVT::f16)
6277 return Register();
6278 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6279 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6280 }
6281 return Register();
6282}
6283
6284Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v8f16_r(MVT RetVT, Register Op0) {
6285 if (RetVT.SimpleTy != MVT::f16)
6286 return Register();
6287 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6288 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6289 }
6290 return Register();
6291}
6292
6293Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f32_r(MVT RetVT, Register Op0) {
6294 if (RetVT.SimpleTy != MVT::f32)
6295 return Register();
6296 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6297}
6298
6299Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f32_r(MVT RetVT, Register Op0) {
6300 if (RetVT.SimpleTy != MVT::f32)
6301 return Register();
6302 if ((Subtarget->isNeonAvailable())) {
6303 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6304 }
6305 return Register();
6306}
6307
6308Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f64_r(MVT RetVT, Register Op0) {
6309 if (RetVT.SimpleTy != MVT::f64)
6310 return Register();
6311 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6312}
6313
6314Register fastEmit_ISD_VECREDUCE_FMAX_r(MVT VT, MVT RetVT, Register Op0) {
6315 switch (VT.SimpleTy) {
6316 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f16_r(RetVT, Op0);
6317 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v8f16_r(RetVT, Op0);
6318 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f32_r(RetVT, Op0);
6319 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f32_r(RetVT, Op0);
6320 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f64_r(RetVT, Op0);
6321 default: return Register();
6322 }
6323}
6324
6325// FastEmit functions for ISD::VECREDUCE_FMAXIMUM.
6326
6327Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f16_r(MVT RetVT, Register Op0) {
6328 if (RetVT.SimpleTy != MVT::f16)
6329 return Register();
6330 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6331 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6332 }
6333 return Register();
6334}
6335
6336Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v8f16_r(MVT RetVT, Register Op0) {
6337 if (RetVT.SimpleTy != MVT::f16)
6338 return Register();
6339 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6340 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6341 }
6342 return Register();
6343}
6344
6345Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f32_r(MVT RetVT, Register Op0) {
6346 if (RetVT.SimpleTy != MVT::f32)
6347 return Register();
6348 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6349}
6350
6351Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f32_r(MVT RetVT, Register Op0) {
6352 if (RetVT.SimpleTy != MVT::f32)
6353 return Register();
6354 if ((Subtarget->isNeonAvailable())) {
6355 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6356 }
6357 return Register();
6358}
6359
6360Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f64_r(MVT RetVT, Register Op0) {
6361 if (RetVT.SimpleTy != MVT::f64)
6362 return Register();
6363 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6364}
6365
6366Register fastEmit_ISD_VECREDUCE_FMAXIMUM_r(MVT VT, MVT RetVT, Register Op0) {
6367 switch (VT.SimpleTy) {
6368 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f16_r(RetVT, Op0);
6369 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v8f16_r(RetVT, Op0);
6370 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f32_r(RetVT, Op0);
6371 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f32_r(RetVT, Op0);
6372 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f64_r(RetVT, Op0);
6373 default: return Register();
6374 }
6375}
6376
6377// FastEmit functions for ISD::VECREDUCE_FMIN.
6378
6379Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f16_r(MVT RetVT, Register Op0) {
6380 if (RetVT.SimpleTy != MVT::f16)
6381 return Register();
6382 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6383 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6384 }
6385 return Register();
6386}
6387
6388Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v8f16_r(MVT RetVT, Register Op0) {
6389 if (RetVT.SimpleTy != MVT::f16)
6390 return Register();
6391 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6392 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6393 }
6394 return Register();
6395}
6396
6397Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f32_r(MVT RetVT, Register Op0) {
6398 if (RetVT.SimpleTy != MVT::f32)
6399 return Register();
6400 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6401}
6402
6403Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f32_r(MVT RetVT, Register Op0) {
6404 if (RetVT.SimpleTy != MVT::f32)
6405 return Register();
6406 if ((Subtarget->isNeonAvailable())) {
6407 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6408 }
6409 return Register();
6410}
6411
6412Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f64_r(MVT RetVT, Register Op0) {
6413 if (RetVT.SimpleTy != MVT::f64)
6414 return Register();
6415 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6416}
6417
6418Register fastEmit_ISD_VECREDUCE_FMIN_r(MVT VT, MVT RetVT, Register Op0) {
6419 switch (VT.SimpleTy) {
6420 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f16_r(RetVT, Op0);
6421 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v8f16_r(RetVT, Op0);
6422 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f32_r(RetVT, Op0);
6423 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f32_r(RetVT, Op0);
6424 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f64_r(RetVT, Op0);
6425 default: return Register();
6426 }
6427}
6428
6429// FastEmit functions for ISD::VECREDUCE_FMINIMUM.
6430
6431Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f16_r(MVT RetVT, Register Op0) {
6432 if (RetVT.SimpleTy != MVT::f16)
6433 return Register();
6434 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6435 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6436 }
6437 return Register();
6438}
6439
6440Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v8f16_r(MVT RetVT, Register Op0) {
6441 if (RetVT.SimpleTy != MVT::f16)
6442 return Register();
6443 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6444 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6445 }
6446 return Register();
6447}
6448
6449Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f32_r(MVT RetVT, Register Op0) {
6450 if (RetVT.SimpleTy != MVT::f32)
6451 return Register();
6452 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6453}
6454
6455Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f32_r(MVT RetVT, Register Op0) {
6456 if (RetVT.SimpleTy != MVT::f32)
6457 return Register();
6458 if ((Subtarget->isNeonAvailable())) {
6459 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6460 }
6461 return Register();
6462}
6463
6464Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f64_r(MVT RetVT, Register Op0) {
6465 if (RetVT.SimpleTy != MVT::f64)
6466 return Register();
6467 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6468}
6469
6470Register fastEmit_ISD_VECREDUCE_FMINIMUM_r(MVT VT, MVT RetVT, Register Op0) {
6471 switch (VT.SimpleTy) {
6472 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f16_r(RetVT, Op0);
6473 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v8f16_r(RetVT, Op0);
6474 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f32_r(RetVT, Op0);
6475 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f32_r(RetVT, Op0);
6476 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f64_r(RetVT, Op0);
6477 default: return Register();
6478 }
6479}
6480
6481// FastEmit functions for ISD::VECREDUCE_SMAX.
6482
6483Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i8_r(MVT RetVT, Register Op0) {
6484 if (RetVT.SimpleTy != MVT::i8)
6485 return Register();
6486 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6487}
6488
6489Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v16i8_r(MVT RetVT, Register Op0) {
6490 if (RetVT.SimpleTy != MVT::i8)
6491 return Register();
6492 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6493}
6494
6495Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i16_r(MVT RetVT, Register Op0) {
6496 if (RetVT.SimpleTy != MVT::i16)
6497 return Register();
6498 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6499}
6500
6501Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i16_r(MVT RetVT, Register Op0) {
6502 if (RetVT.SimpleTy != MVT::i16)
6503 return Register();
6504 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6505}
6506
6507Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i32_r(MVT RetVT, Register Op0) {
6508 if (RetVT.SimpleTy != MVT::i32)
6509 return Register();
6510 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6511}
6512
6513Register fastEmit_ISD_VECREDUCE_SMAX_r(MVT VT, MVT RetVT, Register Op0) {
6514 switch (VT.SimpleTy) {
6515 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i8_r(RetVT, Op0);
6516 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v16i8_r(RetVT, Op0);
6517 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i16_r(RetVT, Op0);
6518 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i16_r(RetVT, Op0);
6519 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i32_r(RetVT, Op0);
6520 default: return Register();
6521 }
6522}
6523
6524// FastEmit functions for ISD::VECREDUCE_SMIN.
6525
6526Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i8_r(MVT RetVT, Register Op0) {
6527 if (RetVT.SimpleTy != MVT::i8)
6528 return Register();
6529 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6530}
6531
6532Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v16i8_r(MVT RetVT, Register Op0) {
6533 if (RetVT.SimpleTy != MVT::i8)
6534 return Register();
6535 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6536}
6537
6538Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i16_r(MVT RetVT, Register Op0) {
6539 if (RetVT.SimpleTy != MVT::i16)
6540 return Register();
6541 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6542}
6543
6544Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i16_r(MVT RetVT, Register Op0) {
6545 if (RetVT.SimpleTy != MVT::i16)
6546 return Register();
6547 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6548}
6549
6550Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i32_r(MVT RetVT, Register Op0) {
6551 if (RetVT.SimpleTy != MVT::i32)
6552 return Register();
6553 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6554}
6555
6556Register fastEmit_ISD_VECREDUCE_SMIN_r(MVT VT, MVT RetVT, Register Op0) {
6557 switch (VT.SimpleTy) {
6558 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i8_r(RetVT, Op0);
6559 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v16i8_r(RetVT, Op0);
6560 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i16_r(RetVT, Op0);
6561 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i16_r(RetVT, Op0);
6562 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i32_r(RetVT, Op0);
6563 default: return Register();
6564 }
6565}
6566
6567// FastEmit functions for ISD::VECREDUCE_UMAX.
6568
6569Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i8_r(MVT RetVT, Register Op0) {
6570 if (RetVT.SimpleTy != MVT::i8)
6571 return Register();
6572 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6573}
6574
6575Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v16i8_r(MVT RetVT, Register Op0) {
6576 if (RetVT.SimpleTy != MVT::i8)
6577 return Register();
6578 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6579}
6580
6581Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i16_r(MVT RetVT, Register Op0) {
6582 if (RetVT.SimpleTy != MVT::i16)
6583 return Register();
6584 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6585}
6586
6587Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i16_r(MVT RetVT, Register Op0) {
6588 if (RetVT.SimpleTy != MVT::i16)
6589 return Register();
6590 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6591}
6592
6593Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i32_r(MVT RetVT, Register Op0) {
6594 if (RetVT.SimpleTy != MVT::i32)
6595 return Register();
6596 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6597}
6598
6599Register fastEmit_ISD_VECREDUCE_UMAX_r(MVT VT, MVT RetVT, Register Op0) {
6600 switch (VT.SimpleTy) {
6601 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i8_r(RetVT, Op0);
6602 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v16i8_r(RetVT, Op0);
6603 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i16_r(RetVT, Op0);
6604 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i16_r(RetVT, Op0);
6605 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i32_r(RetVT, Op0);
6606 default: return Register();
6607 }
6608}
6609
6610// FastEmit functions for ISD::VECREDUCE_UMIN.
6611
6612Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i8_r(MVT RetVT, Register Op0) {
6613 if (RetVT.SimpleTy != MVT::i8)
6614 return Register();
6615 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6616}
6617
6618Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v16i8_r(MVT RetVT, Register Op0) {
6619 if (RetVT.SimpleTy != MVT::i8)
6620 return Register();
6621 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6622}
6623
6624Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i16_r(MVT RetVT, Register Op0) {
6625 if (RetVT.SimpleTy != MVT::i16)
6626 return Register();
6627 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6628}
6629
6630Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i16_r(MVT RetVT, Register Op0) {
6631 if (RetVT.SimpleTy != MVT::i16)
6632 return Register();
6633 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6634}
6635
6636Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i32_r(MVT RetVT, Register Op0) {
6637 if (RetVT.SimpleTy != MVT::i32)
6638 return Register();
6639 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6640}
6641
6642Register fastEmit_ISD_VECREDUCE_UMIN_r(MVT VT, MVT RetVT, Register Op0) {
6643 switch (VT.SimpleTy) {
6644 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i8_r(RetVT, Op0);
6645 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v16i8_r(RetVT, Op0);
6646 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i16_r(RetVT, Op0);
6647 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i16_r(RetVT, Op0);
6648 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i32_r(RetVT, Op0);
6649 default: return Register();
6650 }
6651}
6652
6653// FastEmit functions for ISD::VECTOR_REVERSE.
6654
6655Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(MVT RetVT, Register Op0) {
6656 if (RetVT.SimpleTy != MVT::nxv2i1)
6657 return Register();
6658 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6659 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_D, RC: &AArch64::PPRRegClass, Op0);
6660 }
6661 return Register();
6662}
6663
6664Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(MVT RetVT, Register Op0) {
6665 if (RetVT.SimpleTy != MVT::nxv4i1)
6666 return Register();
6667 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6668 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_S, RC: &AArch64::PPRRegClass, Op0);
6669 }
6670 return Register();
6671}
6672
6673Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(MVT RetVT, Register Op0) {
6674 if (RetVT.SimpleTy != MVT::nxv8i1)
6675 return Register();
6676 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6677 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_H, RC: &AArch64::PPRRegClass, Op0);
6678 }
6679 return Register();
6680}
6681
6682Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(MVT RetVT, Register Op0) {
6683 if (RetVT.SimpleTy != MVT::nxv16i1)
6684 return Register();
6685 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6686 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_B, RC: &AArch64::PPRRegClass, Op0);
6687 }
6688 return Register();
6689}
6690
6691Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
6692 if (RetVT.SimpleTy != MVT::nxv16i8)
6693 return Register();
6694 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6695 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_B, RC: &AArch64::ZPRRegClass, Op0);
6696 }
6697 return Register();
6698}
6699
6700Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
6701 if (RetVT.SimpleTy != MVT::nxv8i16)
6702 return Register();
6703 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6704 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6705 }
6706 return Register();
6707}
6708
6709Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
6710 if (RetVT.SimpleTy != MVT::nxv4i32)
6711 return Register();
6712 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6713 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6714 }
6715 return Register();
6716}
6717
6718Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(MVT RetVT, Register Op0) {
6719 if (RetVT.SimpleTy != MVT::nxv2i64)
6720 return Register();
6721 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6722 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6723 }
6724 return Register();
6725}
6726
6727Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(MVT RetVT, Register Op0) {
6728 if (RetVT.SimpleTy != MVT::nxv2f16)
6729 return Register();
6730 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6731 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6732 }
6733 return Register();
6734}
6735
6736Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(MVT RetVT, Register Op0) {
6737 if (RetVT.SimpleTy != MVT::nxv4f16)
6738 return Register();
6739 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6740 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6741 }
6742 return Register();
6743}
6744
6745Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
6746 if (RetVT.SimpleTy != MVT::nxv8f16)
6747 return Register();
6748 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6749 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6750 }
6751 return Register();
6752}
6753
6754Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(MVT RetVT, Register Op0) {
6755 if (RetVT.SimpleTy != MVT::nxv2bf16)
6756 return Register();
6757 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6758 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6759 }
6760 return Register();
6761}
6762
6763Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(MVT RetVT, Register Op0) {
6764 if (RetVT.SimpleTy != MVT::nxv4bf16)
6765 return Register();
6766 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6767 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6768 }
6769 return Register();
6770}
6771
6772Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(MVT RetVT, Register Op0) {
6773 if (RetVT.SimpleTy != MVT::nxv8bf16)
6774 return Register();
6775 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6776 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6777 }
6778 return Register();
6779}
6780
6781Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(MVT RetVT, Register Op0) {
6782 if (RetVT.SimpleTy != MVT::nxv2f32)
6783 return Register();
6784 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6785 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6786 }
6787 return Register();
6788}
6789
6790Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
6791 if (RetVT.SimpleTy != MVT::nxv4f32)
6792 return Register();
6793 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6794 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6795 }
6796 return Register();
6797}
6798
6799Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
6800 if (RetVT.SimpleTy != MVT::nxv2f64)
6801 return Register();
6802 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6803 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6804 }
6805 return Register();
6806}
6807
6808Register fastEmit_ISD_VECTOR_REVERSE_r(MVT VT, MVT RetVT, Register Op0) {
6809 switch (VT.SimpleTy) {
6810 case MVT::nxv2i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(RetVT, Op0);
6811 case MVT::nxv4i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(RetVT, Op0);
6812 case MVT::nxv8i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(RetVT, Op0);
6813 case MVT::nxv16i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(RetVT, Op0);
6814 case MVT::nxv16i8: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(RetVT, Op0);
6815 case MVT::nxv8i16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(RetVT, Op0);
6816 case MVT::nxv4i32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(RetVT, Op0);
6817 case MVT::nxv2i64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(RetVT, Op0);
6818 case MVT::nxv2f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(RetVT, Op0);
6819 case MVT::nxv4f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(RetVT, Op0);
6820 case MVT::nxv8f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(RetVT, Op0);
6821 case MVT::nxv2bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(RetVT, Op0);
6822 case MVT::nxv4bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(RetVT, Op0);
6823 case MVT::nxv8bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(RetVT, Op0);
6824 case MVT::nxv2f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(RetVT, Op0);
6825 case MVT::nxv4f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(RetVT, Op0);
6826 case MVT::nxv2f64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(RetVT, Op0);
6827 default: return Register();
6828 }
6829}
6830
6831// Top-level FastEmit function.
6832
6833Register fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, Register Op0) override {
6834 switch (Opcode) {
6835 case AArch64ISD::ALLOCATE_ZA_BUFFER: return fastEmit_AArch64ISD_ALLOCATE_ZA_BUFFER_r(VT, RetVT, Op0);
6836 case AArch64ISD::ALLOC_SME_SAVE_BUFFER: return fastEmit_AArch64ISD_ALLOC_SME_SAVE_BUFFER_r(VT, RetVT, Op0);
6837 case AArch64ISD::CALL: return fastEmit_AArch64ISD_CALL_r(VT, RetVT, Op0);
6838 case AArch64ISD::COALESCER_BARRIER: return fastEmit_AArch64ISD_COALESCER_BARRIER_r(VT, RetVT, Op0);
6839 case AArch64ISD::DUP: return fastEmit_AArch64ISD_DUP_r(VT, RetVT, Op0);
6840 case AArch64ISD::FCVTXN: return fastEmit_AArch64ISD_FCVTXN_r(VT, RetVT, Op0);
6841 case AArch64ISD::FRECPE: return fastEmit_AArch64ISD_FRECPE_r(VT, RetVT, Op0);
6842 case AArch64ISD::FRSQRTE: return fastEmit_AArch64ISD_FRSQRTE_r(VT, RetVT, Op0);
6843 case AArch64ISD::PROBED_ALLOCA: return fastEmit_AArch64ISD_PROBED_ALLOCA_r(VT, RetVT, Op0);
6844 case AArch64ISD::REV16: return fastEmit_AArch64ISD_REV16_r(VT, RetVT, Op0);
6845 case AArch64ISD::REV32: return fastEmit_AArch64ISD_REV32_r(VT, RetVT, Op0);
6846 case AArch64ISD::REV64: return fastEmit_AArch64ISD_REV64_r(VT, RetVT, Op0);
6847 case AArch64ISD::SADDLP: return fastEmit_AArch64ISD_SADDLP_r(VT, RetVT, Op0);
6848 case AArch64ISD::SITOF: return fastEmit_AArch64ISD_SITOF_r(VT, RetVT, Op0);
6849 case AArch64ISD::SUNPKHI: return fastEmit_AArch64ISD_SUNPKHI_r(VT, RetVT, Op0);
6850 case AArch64ISD::SUNPKLO: return fastEmit_AArch64ISD_SUNPKLO_r(VT, RetVT, Op0);
6851 case AArch64ISD::UADDLP: return fastEmit_AArch64ISD_UADDLP_r(VT, RetVT, Op0);
6852 case AArch64ISD::UITOF: return fastEmit_AArch64ISD_UITOF_r(VT, RetVT, Op0);
6853 case AArch64ISD::UUNPKHI: return fastEmit_AArch64ISD_UUNPKHI_r(VT, RetVT, Op0);
6854 case AArch64ISD::UUNPKLO: return fastEmit_AArch64ISD_UUNPKLO_r(VT, RetVT, Op0);
6855 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
6856 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
6857 case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0);
6858 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0);
6859 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
6860 case ISD::CTLS: return fastEmit_ISD_CTLS_r(VT, RetVT, Op0);
6861 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
6862 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
6863 case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0);
6864 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
6865 case ISD::FCEIL: return fastEmit_ISD_FCEIL_r(VT, RetVT, Op0);
6866 case ISD::FFLOOR: return fastEmit_ISD_FFLOOR_r(VT, RetVT, Op0);
6867 case ISD::FNEARBYINT: return fastEmit_ISD_FNEARBYINT_r(VT, RetVT, Op0);
6868 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
6869 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
6870 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
6871 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
6872 case ISD::FP_TO_SINT_SAT: return fastEmit_ISD_FP_TO_SINT_SAT_r(VT, RetVT, Op0);
6873 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
6874 case ISD::FP_TO_UINT_SAT: return fastEmit_ISD_FP_TO_UINT_SAT_r(VT, RetVT, Op0);
6875 case ISD::FRINT: return fastEmit_ISD_FRINT_r(VT, RetVT, Op0);
6876 case ISD::FROUND: return fastEmit_ISD_FROUND_r(VT, RetVT, Op0);
6877 case ISD::FROUNDEVEN: return fastEmit_ISD_FROUNDEVEN_r(VT, RetVT, Op0);
6878 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
6879 case ISD::FTRUNC: return fastEmit_ISD_FTRUNC_r(VT, RetVT, Op0);
6880 case ISD::LLROUND: return fastEmit_ISD_LLROUND_r(VT, RetVT, Op0);
6881 case ISD::LROUND: return fastEmit_ISD_LROUND_r(VT, RetVT, Op0);
6882 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
6883 case ISD::SPLAT_VECTOR: return fastEmit_ISD_SPLAT_VECTOR_r(VT, RetVT, Op0);
6884 case ISD::STRICT_FCEIL: return fastEmit_ISD_STRICT_FCEIL_r(VT, RetVT, Op0);
6885 case ISD::STRICT_FFLOOR: return fastEmit_ISD_STRICT_FFLOOR_r(VT, RetVT, Op0);
6886 case ISD::STRICT_FNEARBYINT: return fastEmit_ISD_STRICT_FNEARBYINT_r(VT, RetVT, Op0);
6887 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0);
6888 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0);
6889 case ISD::STRICT_FP_TO_SINT: return fastEmit_ISD_STRICT_FP_TO_SINT_r(VT, RetVT, Op0);
6890 case ISD::STRICT_FP_TO_UINT: return fastEmit_ISD_STRICT_FP_TO_UINT_r(VT, RetVT, Op0);
6891 case ISD::STRICT_FRINT: return fastEmit_ISD_STRICT_FRINT_r(VT, RetVT, Op0);
6892 case ISD::STRICT_FROUND: return fastEmit_ISD_STRICT_FROUND_r(VT, RetVT, Op0);
6893 case ISD::STRICT_FROUNDEVEN: return fastEmit_ISD_STRICT_FROUNDEVEN_r(VT, RetVT, Op0);
6894 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0);
6895 case ISD::STRICT_FTRUNC: return fastEmit_ISD_STRICT_FTRUNC_r(VT, RetVT, Op0);
6896 case ISD::STRICT_LLROUND: return fastEmit_ISD_STRICT_LLROUND_r(VT, RetVT, Op0);
6897 case ISD::STRICT_LROUND: return fastEmit_ISD_STRICT_LROUND_r(VT, RetVT, Op0);
6898 case ISD::STRICT_SINT_TO_FP: return fastEmit_ISD_STRICT_SINT_TO_FP_r(VT, RetVT, Op0);
6899 case ISD::STRICT_UINT_TO_FP: return fastEmit_ISD_STRICT_UINT_TO_FP_r(VT, RetVT, Op0);
6900 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
6901 case ISD::TRUNCATE_SSAT_S: return fastEmit_ISD_TRUNCATE_SSAT_S_r(VT, RetVT, Op0);
6902 case ISD::TRUNCATE_SSAT_U: return fastEmit_ISD_TRUNCATE_SSAT_U_r(VT, RetVT, Op0);
6903 case ISD::TRUNCATE_USAT_U: return fastEmit_ISD_TRUNCATE_USAT_U_r(VT, RetVT, Op0);
6904 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
6905 case ISD::VECREDUCE_ADD: return fastEmit_ISD_VECREDUCE_ADD_r(VT, RetVT, Op0);
6906 case ISD::VECREDUCE_FADD: return fastEmit_ISD_VECREDUCE_FADD_r(VT, RetVT, Op0);
6907 case ISD::VECREDUCE_FMAX: return fastEmit_ISD_VECREDUCE_FMAX_r(VT, RetVT, Op0);
6908 case ISD::VECREDUCE_FMAXIMUM: return fastEmit_ISD_VECREDUCE_FMAXIMUM_r(VT, RetVT, Op0);
6909 case ISD::VECREDUCE_FMIN: return fastEmit_ISD_VECREDUCE_FMIN_r(VT, RetVT, Op0);
6910 case ISD::VECREDUCE_FMINIMUM: return fastEmit_ISD_VECREDUCE_FMINIMUM_r(VT, RetVT, Op0);
6911 case ISD::VECREDUCE_SMAX: return fastEmit_ISD_VECREDUCE_SMAX_r(VT, RetVT, Op0);
6912 case ISD::VECREDUCE_SMIN: return fastEmit_ISD_VECREDUCE_SMIN_r(VT, RetVT, Op0);
6913 case ISD::VECREDUCE_UMAX: return fastEmit_ISD_VECREDUCE_UMAX_r(VT, RetVT, Op0);
6914 case ISD::VECREDUCE_UMIN: return fastEmit_ISD_VECREDUCE_UMIN_r(VT, RetVT, Op0);
6915 case ISD::VECTOR_REVERSE: return fastEmit_ISD_VECTOR_REVERSE_r(VT, RetVT, Op0);
6916 default: return Register();
6917 }
6918}
6919
6920// FastEmit functions for AArch64ISD::ADDP.
6921
6922Register fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6923 if (RetVT.SimpleTy != MVT::v8i8)
6924 return Register();
6925 if ((Subtarget->isNeonAvailable())) {
6926 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
6927 }
6928 return Register();
6929}
6930
6931Register fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6932 if (RetVT.SimpleTy != MVT::v16i8)
6933 return Register();
6934 if ((Subtarget->isNeonAvailable())) {
6935 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
6936 }
6937 return Register();
6938}
6939
6940Register fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6941 if (RetVT.SimpleTy != MVT::v4i16)
6942 return Register();
6943 if ((Subtarget->isNeonAvailable())) {
6944 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6945 }
6946 return Register();
6947}
6948
6949Register fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6950 if (RetVT.SimpleTy != MVT::v8i16)
6951 return Register();
6952 if ((Subtarget->isNeonAvailable())) {
6953 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
6954 }
6955 return Register();
6956}
6957
6958Register fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6959 if (RetVT.SimpleTy != MVT::v2i32)
6960 return Register();
6961 if ((Subtarget->isNeonAvailable())) {
6962 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6963 }
6964 return Register();
6965}
6966
6967Register fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6968 if (RetVT.SimpleTy != MVT::v4i32)
6969 return Register();
6970 if ((Subtarget->isNeonAvailable())) {
6971 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
6972 }
6973 return Register();
6974}
6975
6976Register fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6977 if (RetVT.SimpleTy != MVT::v2i64)
6978 return Register();
6979 if ((Subtarget->isNeonAvailable())) {
6980 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
6981 }
6982 return Register();
6983}
6984
6985Register fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
6986 if (RetVT.SimpleTy != MVT::v4f16)
6987 return Register();
6988 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6989 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6990 }
6991 return Register();
6992}
6993
6994Register fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
6995 if (RetVT.SimpleTy != MVT::v8f16)
6996 return Register();
6997 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6998 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
6999 }
7000 return Register();
7001}
7002
7003Register fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7004 if (RetVT.SimpleTy != MVT::v2f32)
7005 return Register();
7006 if ((Subtarget->isNeonAvailable())) {
7007 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7008 }
7009 return Register();
7010}
7011
7012Register fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7013 if (RetVT.SimpleTy != MVT::v4f32)
7014 return Register();
7015 if ((Subtarget->isNeonAvailable())) {
7016 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7017 }
7018 return Register();
7019}
7020
7021Register fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7022 if (RetVT.SimpleTy != MVT::v2f64)
7023 return Register();
7024 if ((Subtarget->isNeonAvailable())) {
7025 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7026 }
7027 return Register();
7028}
7029
7030Register fastEmit_AArch64ISD_ADDP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7031 switch (VT.SimpleTy) {
7032 case MVT::v8i8: return fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(RetVT, Op0, Op1);
7033 case MVT::v16i8: return fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(RetVT, Op0, Op1);
7034 case MVT::v4i16: return fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(RetVT, Op0, Op1);
7035 case MVT::v8i16: return fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(RetVT, Op0, Op1);
7036 case MVT::v2i32: return fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(RetVT, Op0, Op1);
7037 case MVT::v4i32: return fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(RetVT, Op0, Op1);
7038 case MVT::v2i64: return fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(RetVT, Op0, Op1);
7039 case MVT::v4f16: return fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(RetVT, Op0, Op1);
7040 case MVT::v8f16: return fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(RetVT, Op0, Op1);
7041 case MVT::v2f32: return fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(RetVT, Op0, Op1);
7042 case MVT::v4f32: return fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(RetVT, Op0, Op1);
7043 case MVT::v2f64: return fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(RetVT, Op0, Op1);
7044 default: return Register();
7045 }
7046}
7047
7048// FastEmit functions for AArch64ISD::BIC.
7049
7050Register fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7051 if (RetVT.SimpleTy != MVT::nxv16i8)
7052 return Register();
7053 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7054 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
7055 }
7056 return Register();
7057}
7058
7059Register fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7060 if (RetVT.SimpleTy != MVT::nxv8i16)
7061 return Register();
7062 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7063 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
7064 }
7065 return Register();
7066}
7067
7068Register fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7069 if (RetVT.SimpleTy != MVT::nxv4i32)
7070 return Register();
7071 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7072 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
7073 }
7074 return Register();
7075}
7076
7077Register fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7078 if (RetVT.SimpleTy != MVT::nxv2i64)
7079 return Register();
7080 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7081 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
7082 }
7083 return Register();
7084}
7085
7086Register fastEmit_AArch64ISD_BIC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7087 switch (VT.SimpleTy) {
7088 case MVT::nxv16i8: return fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(RetVT, Op0, Op1);
7089 case MVT::nxv8i16: return fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(RetVT, Op0, Op1);
7090 case MVT::nxv4i32: return fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(RetVT, Op0, Op1);
7091 case MVT::nxv2i64: return fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(RetVT, Op0, Op1);
7092 default: return Register();
7093 }
7094}
7095
7096// FastEmit functions for AArch64ISD::FCMEQ.
7097
7098Register fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7099 if (RetVT.SimpleTy != MVT::i32)
7100 return Register();
7101 if ((Subtarget->isNeonAvailable())) {
7102 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7103 }
7104 return Register();
7105}
7106
7107Register fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7108 if (RetVT.SimpleTy != MVT::i64)
7109 return Register();
7110 if ((Subtarget->isNeonAvailable())) {
7111 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7112 }
7113 return Register();
7114}
7115
7116Register fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7117 if (RetVT.SimpleTy != MVT::v4i16)
7118 return Register();
7119 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7120 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7121 }
7122 return Register();
7123}
7124
7125Register fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7126 if (RetVT.SimpleTy != MVT::v8i16)
7127 return Register();
7128 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7129 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7130 }
7131 return Register();
7132}
7133
7134Register fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7135 if (RetVT.SimpleTy != MVT::v2i32)
7136 return Register();
7137 if ((Subtarget->isNeonAvailable())) {
7138 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7139 }
7140 return Register();
7141}
7142
7143Register fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7144 if (RetVT.SimpleTy != MVT::v4i32)
7145 return Register();
7146 if ((Subtarget->isNeonAvailable())) {
7147 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7148 }
7149 return Register();
7150}
7151
7152Register fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7153 if (RetVT.SimpleTy != MVT::v1i64)
7154 return Register();
7155 if ((Subtarget->isNeonAvailable())) {
7156 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7157 }
7158 return Register();
7159}
7160
7161Register fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7162 if (RetVT.SimpleTy != MVT::v2i64)
7163 return Register();
7164 if ((Subtarget->isNeonAvailable())) {
7165 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7166 }
7167 return Register();
7168}
7169
7170Register fastEmit_AArch64ISD_FCMEQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7171 switch (VT.SimpleTy) {
7172 case MVT::f32: return fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(RetVT, Op0, Op1);
7173 case MVT::f64: return fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(RetVT, Op0, Op1);
7174 case MVT::v4f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(RetVT, Op0, Op1);
7175 case MVT::v8f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(RetVT, Op0, Op1);
7176 case MVT::v2f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(RetVT, Op0, Op1);
7177 case MVT::v4f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(RetVT, Op0, Op1);
7178 case MVT::v1f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(RetVT, Op0, Op1);
7179 case MVT::v2f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(RetVT, Op0, Op1);
7180 default: return Register();
7181 }
7182}
7183
7184// FastEmit functions for AArch64ISD::FCMGE.
7185
7186Register fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7187 if (RetVT.SimpleTy != MVT::i32)
7188 return Register();
7189 if ((Subtarget->isNeonAvailable())) {
7190 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7191 }
7192 return Register();
7193}
7194
7195Register fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7196 if (RetVT.SimpleTy != MVT::i64)
7197 return Register();
7198 if ((Subtarget->isNeonAvailable())) {
7199 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7200 }
7201 return Register();
7202}
7203
7204Register fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7205 if (RetVT.SimpleTy != MVT::v4i16)
7206 return Register();
7207 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7208 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7209 }
7210 return Register();
7211}
7212
7213Register fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7214 if (RetVT.SimpleTy != MVT::v8i16)
7215 return Register();
7216 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7217 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7218 }
7219 return Register();
7220}
7221
7222Register fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7223 if (RetVT.SimpleTy != MVT::v2i32)
7224 return Register();
7225 if ((Subtarget->isNeonAvailable())) {
7226 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7227 }
7228 return Register();
7229}
7230
7231Register fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7232 if (RetVT.SimpleTy != MVT::v4i32)
7233 return Register();
7234 if ((Subtarget->isNeonAvailable())) {
7235 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7236 }
7237 return Register();
7238}
7239
7240Register fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7241 if (RetVT.SimpleTy != MVT::v1i64)
7242 return Register();
7243 if ((Subtarget->isNeonAvailable())) {
7244 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7245 }
7246 return Register();
7247}
7248
7249Register fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7250 if (RetVT.SimpleTy != MVT::v2i64)
7251 return Register();
7252 if ((Subtarget->isNeonAvailable())) {
7253 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7254 }
7255 return Register();
7256}
7257
7258Register fastEmit_AArch64ISD_FCMGE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7259 switch (VT.SimpleTy) {
7260 case MVT::f32: return fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(RetVT, Op0, Op1);
7261 case MVT::f64: return fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(RetVT, Op0, Op1);
7262 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(RetVT, Op0, Op1);
7263 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(RetVT, Op0, Op1);
7264 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(RetVT, Op0, Op1);
7265 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(RetVT, Op0, Op1);
7266 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(RetVT, Op0, Op1);
7267 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(RetVT, Op0, Op1);
7268 default: return Register();
7269 }
7270}
7271
7272// FastEmit functions for AArch64ISD::FCMGT.
7273
7274Register fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7275 if (RetVT.SimpleTy != MVT::i32)
7276 return Register();
7277 if ((Subtarget->isNeonAvailable())) {
7278 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7279 }
7280 return Register();
7281}
7282
7283Register fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7284 if (RetVT.SimpleTy != MVT::i64)
7285 return Register();
7286 if ((Subtarget->isNeonAvailable())) {
7287 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7288 }
7289 return Register();
7290}
7291
7292Register fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7293 if (RetVT.SimpleTy != MVT::v4i16)
7294 return Register();
7295 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7296 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7297 }
7298 return Register();
7299}
7300
7301Register fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7302 if (RetVT.SimpleTy != MVT::v8i16)
7303 return Register();
7304 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7305 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7306 }
7307 return Register();
7308}
7309
7310Register fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7311 if (RetVT.SimpleTy != MVT::v2i32)
7312 return Register();
7313 if ((Subtarget->isNeonAvailable())) {
7314 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7315 }
7316 return Register();
7317}
7318
7319Register fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7320 if (RetVT.SimpleTy != MVT::v4i32)
7321 return Register();
7322 if ((Subtarget->isNeonAvailable())) {
7323 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7324 }
7325 return Register();
7326}
7327
7328Register fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7329 if (RetVT.SimpleTy != MVT::v1i64)
7330 return Register();
7331 if ((Subtarget->isNeonAvailable())) {
7332 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7333 }
7334 return Register();
7335}
7336
7337Register fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7338 if (RetVT.SimpleTy != MVT::v2i64)
7339 return Register();
7340 if ((Subtarget->isNeonAvailable())) {
7341 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7342 }
7343 return Register();
7344}
7345
7346Register fastEmit_AArch64ISD_FCMGT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7347 switch (VT.SimpleTy) {
7348 case MVT::f32: return fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(RetVT, Op0, Op1);
7349 case MVT::f64: return fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(RetVT, Op0, Op1);
7350 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(RetVT, Op0, Op1);
7351 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(RetVT, Op0, Op1);
7352 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(RetVT, Op0, Op1);
7353 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(RetVT, Op0, Op1);
7354 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(RetVT, Op0, Op1);
7355 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(RetVT, Op0, Op1);
7356 default: return Register();
7357 }
7358}
7359
7360// FastEmit functions for AArch64ISD::FCMP.
7361
7362Register fastEmit_AArch64ISD_FCMP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
7363 if (RetVT.SimpleTy != MVT::i32)
7364 return Register();
7365 if ((Subtarget->hasFullFP16())) {
7366 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
7367 }
7368 return Register();
7369}
7370
7371Register fastEmit_AArch64ISD_FCMP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7372 if (RetVT.SimpleTy != MVT::i32)
7373 return Register();
7374 if ((Subtarget->hasFPARMv8())) {
7375 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
7376 }
7377 return Register();
7378}
7379
7380Register fastEmit_AArch64ISD_FCMP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7381 if (RetVT.SimpleTy != MVT::i32)
7382 return Register();
7383 if ((Subtarget->hasFPARMv8())) {
7384 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
7385 }
7386 return Register();
7387}
7388
7389Register fastEmit_AArch64ISD_FCMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7390 switch (VT.SimpleTy) {
7391 case MVT::f16: return fastEmit_AArch64ISD_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
7392 case MVT::f32: return fastEmit_AArch64ISD_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
7393 case MVT::f64: return fastEmit_AArch64ISD_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
7394 default: return Register();
7395 }
7396}
7397
7398// FastEmit functions for AArch64ISD::FRECPS.
7399
7400Register fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7401 if (RetVT.SimpleTy != MVT::f32)
7402 return Register();
7403 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7404}
7405
7406Register fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7407 if (RetVT.SimpleTy != MVT::f64)
7408 return Register();
7409 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7410}
7411
7412Register fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7413 if (RetVT.SimpleTy != MVT::v2f32)
7414 return Register();
7415 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7416}
7417
7418Register fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7419 if (RetVT.SimpleTy != MVT::v4f32)
7420 return Register();
7421 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7422}
7423
7424Register fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7425 if (RetVT.SimpleTy != MVT::v2f64)
7426 return Register();
7427 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7428}
7429
7430Register fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7431 if (RetVT.SimpleTy != MVT::nxv8f16)
7432 return Register();
7433 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7434 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7435 }
7436 return Register();
7437}
7438
7439Register fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7440 if (RetVT.SimpleTy != MVT::nxv4f32)
7441 return Register();
7442 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7443 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7444 }
7445 return Register();
7446}
7447
7448Register fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7449 if (RetVT.SimpleTy != MVT::nxv2f64)
7450 return Register();
7451 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7452 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7453 }
7454 return Register();
7455}
7456
7457Register fastEmit_AArch64ISD_FRECPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7458 switch (VT.SimpleTy) {
7459 case MVT::f32: return fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(RetVT, Op0, Op1);
7460 case MVT::f64: return fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(RetVT, Op0, Op1);
7461 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(RetVT, Op0, Op1);
7462 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(RetVT, Op0, Op1);
7463 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(RetVT, Op0, Op1);
7464 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(RetVT, Op0, Op1);
7465 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(RetVT, Op0, Op1);
7466 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(RetVT, Op0, Op1);
7467 default: return Register();
7468 }
7469}
7470
7471// FastEmit functions for AArch64ISD::FRSQRTS.
7472
7473Register fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7474 if (RetVT.SimpleTy != MVT::f32)
7475 return Register();
7476 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7477}
7478
7479Register fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7480 if (RetVT.SimpleTy != MVT::f64)
7481 return Register();
7482 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7483}
7484
7485Register fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7486 if (RetVT.SimpleTy != MVT::v2f32)
7487 return Register();
7488 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7489}
7490
7491Register fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7492 if (RetVT.SimpleTy != MVT::v4f32)
7493 return Register();
7494 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7495}
7496
7497Register fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7498 if (RetVT.SimpleTy != MVT::v2f64)
7499 return Register();
7500 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7501}
7502
7503Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7504 if (RetVT.SimpleTy != MVT::nxv8f16)
7505 return Register();
7506 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7507 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7508 }
7509 return Register();
7510}
7511
7512Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7513 if (RetVT.SimpleTy != MVT::nxv4f32)
7514 return Register();
7515 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7516 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7517 }
7518 return Register();
7519}
7520
7521Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7522 if (RetVT.SimpleTy != MVT::nxv2f64)
7523 return Register();
7524 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7525 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7526 }
7527 return Register();
7528}
7529
7530Register fastEmit_AArch64ISD_FRSQRTS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7531 switch (VT.SimpleTy) {
7532 case MVT::f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(RetVT, Op0, Op1);
7533 case MVT::f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(RetVT, Op0, Op1);
7534 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(RetVT, Op0, Op1);
7535 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(RetVT, Op0, Op1);
7536 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(RetVT, Op0, Op1);
7537 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(RetVT, Op0, Op1);
7538 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(RetVT, Op0, Op1);
7539 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(RetVT, Op0, Op1);
7540 default: return Register();
7541 }
7542}
7543
7544// FastEmit functions for AArch64ISD::INIT_TPIDR2OBJ.
7545
7546Register fastEmit_AArch64ISD_INIT_TPIDR2OBJ_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
7547 if (RetVT.SimpleTy != MVT::isVoid)
7548 return Register();
7549 return fastEmitInst_rr(MachineInstOpcode: AArch64::InitTPIDR2Obj, RC: &AArch64::GPR64RegClass, Op0, Op1);
7550}
7551
7552Register fastEmit_AArch64ISD_INIT_TPIDR2OBJ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7553 switch (VT.SimpleTy) {
7554 case MVT::i64: return fastEmit_AArch64ISD_INIT_TPIDR2OBJ_MVT_i64_rr(RetVT, Op0, Op1);
7555 default: return Register();
7556 }
7557}
7558
7559// FastEmit functions for AArch64ISD::PMULL.
7560
7561Register fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7562 if (RetVT.SimpleTy != MVT::v8i16)
7563 return Register();
7564 if ((Subtarget->isNeonAvailable())) {
7565 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULLv8i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
7566 }
7567 return Register();
7568}
7569
7570Register fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7571 if (RetVT.SimpleTy != MVT::v16i8)
7572 return Register();
7573 if ((Subtarget->hasAES())) {
7574 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULLv1i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7575 }
7576 return Register();
7577}
7578
7579Register fastEmit_AArch64ISD_PMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7580 switch (VT.SimpleTy) {
7581 case MVT::v8i8: return fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
7582 case MVT::v1i64: return fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(RetVT, Op0, Op1);
7583 default: return Register();
7584 }
7585}
7586
7587// FastEmit functions for AArch64ISD::PTEST.
7588
7589Register fastEmit_AArch64ISD_PTEST_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7590 if (RetVT.SimpleTy != MVT::i32)
7591 return Register();
7592 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7593 return fastEmitInst_rr(MachineInstOpcode: AArch64::PTEST_PP, RC: &AArch64::PPRRegClass, Op0, Op1);
7594 }
7595 return Register();
7596}
7597
7598Register fastEmit_AArch64ISD_PTEST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7599 switch (VT.SimpleTy) {
7600 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7601 default: return Register();
7602 }
7603}
7604
7605// FastEmit functions for AArch64ISD::PTEST_ANY.
7606
7607Register fastEmit_AArch64ISD_PTEST_ANY_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7608 if (RetVT.SimpleTy != MVT::i32)
7609 return Register();
7610 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7611 return fastEmitInst_rr(MachineInstOpcode: AArch64::PTEST_PP_ANY, RC: &AArch64::PPRRegClass, Op0, Op1);
7612 }
7613 return Register();
7614}
7615
7616Register fastEmit_AArch64ISD_PTEST_ANY_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7617 switch (VT.SimpleTy) {
7618 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_ANY_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7619 default: return Register();
7620 }
7621}
7622
7623// FastEmit functions for AArch64ISD::PTEST_FIRST.
7624
7625Register fastEmit_AArch64ISD_PTEST_FIRST_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7626 if (RetVT.SimpleTy != MVT::i32)
7627 return Register();
7628 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7629 return fastEmitInst_rr(MachineInstOpcode: AArch64::PTEST_PP_FIRST, RC: &AArch64::PPRRegClass, Op0, Op1);
7630 }
7631 return Register();
7632}
7633
7634Register fastEmit_AArch64ISD_PTEST_FIRST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7635 switch (VT.SimpleTy) {
7636 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_FIRST_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7637 default: return Register();
7638 }
7639}
7640
7641// FastEmit functions for AArch64ISD::SMULL.
7642
7643Register fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7644 if (RetVT.SimpleTy != MVT::v8i16)
7645 return Register();
7646 if ((Subtarget->isNeonAvailable())) {
7647 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv8i8_v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7648 }
7649 return Register();
7650}
7651
7652Register fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7653 if (RetVT.SimpleTy != MVT::v4i32)
7654 return Register();
7655 if ((Subtarget->isNeonAvailable())) {
7656 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv4i16_v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7657 }
7658 return Register();
7659}
7660
7661Register fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7662 if (RetVT.SimpleTy != MVT::v2i64)
7663 return Register();
7664 if ((Subtarget->isNeonAvailable())) {
7665 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv2i32_v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7666 }
7667 return Register();
7668}
7669
7670Register fastEmit_AArch64ISD_SMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7671 switch (VT.SimpleTy) {
7672 case MVT::v8i8: return fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
7673 case MVT::v4i16: return fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(RetVT, Op0, Op1);
7674 case MVT::v2i32: return fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(RetVT, Op0, Op1);
7675 default: return Register();
7676 }
7677}
7678
7679// FastEmit functions for AArch64ISD::SQADD.
7680
7681Register fastEmit_AArch64ISD_SQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7682 if (RetVT.SimpleTy != MVT::f32)
7683 return Register();
7684 if ((Subtarget->isNeonAvailable())) {
7685 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7686 }
7687 return Register();
7688}
7689
7690Register fastEmit_AArch64ISD_SQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7691 if (RetVT.SimpleTy != MVT::f64)
7692 return Register();
7693 if ((Subtarget->isNeonAvailable())) {
7694 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7695 }
7696 return Register();
7697}
7698
7699Register fastEmit_AArch64ISD_SQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7700 switch (VT.SimpleTy) {
7701 case MVT::f32: return fastEmit_AArch64ISD_SQADD_MVT_f32_rr(RetVT, Op0, Op1);
7702 case MVT::f64: return fastEmit_AArch64ISD_SQADD_MVT_f64_rr(RetVT, Op0, Op1);
7703 default: return Register();
7704 }
7705}
7706
7707// FastEmit functions for AArch64ISD::SQDMULH.
7708
7709Register fastEmit_AArch64ISD_SQDMULH_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7710 if (RetVT.SimpleTy != MVT::f32)
7711 return Register();
7712 if ((Subtarget->isNeonAvailable())) {
7713 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7714 }
7715 return Register();
7716}
7717
7718Register fastEmit_AArch64ISD_SQDMULH_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7719 if (RetVT.SimpleTy != MVT::v4i16)
7720 return Register();
7721 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7722}
7723
7724Register fastEmit_AArch64ISD_SQDMULH_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7725 if (RetVT.SimpleTy != MVT::v8i16)
7726 return Register();
7727 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7728}
7729
7730Register fastEmit_AArch64ISD_SQDMULH_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7731 if (RetVT.SimpleTy != MVT::v2i32)
7732 return Register();
7733 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7734}
7735
7736Register fastEmit_AArch64ISD_SQDMULH_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7737 if (RetVT.SimpleTy != MVT::v4i32)
7738 return Register();
7739 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7740}
7741
7742Register fastEmit_AArch64ISD_SQDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7743 switch (VT.SimpleTy) {
7744 case MVT::f32: return fastEmit_AArch64ISD_SQDMULH_MVT_f32_rr(RetVT, Op0, Op1);
7745 case MVT::v4i16: return fastEmit_AArch64ISD_SQDMULH_MVT_v4i16_rr(RetVT, Op0, Op1);
7746 case MVT::v8i16: return fastEmit_AArch64ISD_SQDMULH_MVT_v8i16_rr(RetVT, Op0, Op1);
7747 case MVT::v2i32: return fastEmit_AArch64ISD_SQDMULH_MVT_v2i32_rr(RetVT, Op0, Op1);
7748 case MVT::v4i32: return fastEmit_AArch64ISD_SQDMULH_MVT_v4i32_rr(RetVT, Op0, Op1);
7749 default: return Register();
7750 }
7751}
7752
7753// FastEmit functions for AArch64ISD::SQDMULL.
7754
7755Register fastEmit_AArch64ISD_SQDMULL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7756 if (RetVT.SimpleTy != MVT::f64)
7757 return Register();
7758 if ((Subtarget->isNeonAvailable())) {
7759 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULLi32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7760 }
7761 return Register();
7762}
7763
7764Register fastEmit_AArch64ISD_SQDMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7765 switch (VT.SimpleTy) {
7766 case MVT::f32: return fastEmit_AArch64ISD_SQDMULL_MVT_f32_rr(RetVT, Op0, Op1);
7767 default: return Register();
7768 }
7769}
7770
7771// FastEmit functions for AArch64ISD::SQRDMULH.
7772
7773Register fastEmit_AArch64ISD_SQRDMULH_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7774 if (RetVT.SimpleTy != MVT::f32)
7775 return Register();
7776 if ((Subtarget->isNeonAvailable())) {
7777 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRDMULHv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7778 }
7779 return Register();
7780}
7781
7782Register fastEmit_AArch64ISD_SQRDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7783 switch (VT.SimpleTy) {
7784 case MVT::f32: return fastEmit_AArch64ISD_SQRDMULH_MVT_f32_rr(RetVT, Op0, Op1);
7785 default: return Register();
7786 }
7787}
7788
7789// FastEmit functions for AArch64ISD::SQRSHL.
7790
7791Register fastEmit_AArch64ISD_SQRSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7792 if (RetVT.SimpleTy != MVT::f32)
7793 return Register();
7794 if ((Subtarget->isNeonAvailable())) {
7795 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7796 }
7797 return Register();
7798}
7799
7800Register fastEmit_AArch64ISD_SQRSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7801 if (RetVT.SimpleTy != MVT::f64)
7802 return Register();
7803 if ((Subtarget->isNeonAvailable())) {
7804 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7805 }
7806 return Register();
7807}
7808
7809Register fastEmit_AArch64ISD_SQRSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7810 switch (VT.SimpleTy) {
7811 case MVT::f32: return fastEmit_AArch64ISD_SQRSHL_MVT_f32_rr(RetVT, Op0, Op1);
7812 case MVT::f64: return fastEmit_AArch64ISD_SQRSHL_MVT_f64_rr(RetVT, Op0, Op1);
7813 default: return Register();
7814 }
7815}
7816
7817// FastEmit functions for AArch64ISD::SQSHL.
7818
7819Register fastEmit_AArch64ISD_SQSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7820 if (RetVT.SimpleTy != MVT::f32)
7821 return Register();
7822 if ((Subtarget->isNeonAvailable())) {
7823 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7824 }
7825 return Register();
7826}
7827
7828Register fastEmit_AArch64ISD_SQSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7829 if (RetVT.SimpleTy != MVT::f64)
7830 return Register();
7831 if ((Subtarget->isNeonAvailable())) {
7832 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7833 }
7834 return Register();
7835}
7836
7837Register fastEmit_AArch64ISD_SQSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7838 switch (VT.SimpleTy) {
7839 case MVT::f32: return fastEmit_AArch64ISD_SQSHL_MVT_f32_rr(RetVT, Op0, Op1);
7840 case MVT::f64: return fastEmit_AArch64ISD_SQSHL_MVT_f64_rr(RetVT, Op0, Op1);
7841 default: return Register();
7842 }
7843}
7844
7845// FastEmit functions for AArch64ISD::SQSUB.
7846
7847Register fastEmit_AArch64ISD_SQSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7848 if (RetVT.SimpleTy != MVT::f32)
7849 return Register();
7850 if ((Subtarget->isNeonAvailable())) {
7851 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7852 }
7853 return Register();
7854}
7855
7856Register fastEmit_AArch64ISD_SQSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7857 if (RetVT.SimpleTy != MVT::f64)
7858 return Register();
7859 if ((Subtarget->isNeonAvailable())) {
7860 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7861 }
7862 return Register();
7863}
7864
7865Register fastEmit_AArch64ISD_SQSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7866 switch (VT.SimpleTy) {
7867 case MVT::f32: return fastEmit_AArch64ISD_SQSUB_MVT_f32_rr(RetVT, Op0, Op1);
7868 case MVT::f64: return fastEmit_AArch64ISD_SQSUB_MVT_f64_rr(RetVT, Op0, Op1);
7869 default: return Register();
7870 }
7871}
7872
7873// FastEmit functions for AArch64ISD::STRICT_FCMP.
7874
7875Register fastEmit_AArch64ISD_STRICT_FCMP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
7876 if (RetVT.SimpleTy != MVT::i32)
7877 return Register();
7878 if ((Subtarget->hasFullFP16())) {
7879 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
7880 }
7881 return Register();
7882}
7883
7884Register fastEmit_AArch64ISD_STRICT_FCMP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7885 if (RetVT.SimpleTy != MVT::i32)
7886 return Register();
7887 if ((Subtarget->hasFPARMv8())) {
7888 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
7889 }
7890 return Register();
7891}
7892
7893Register fastEmit_AArch64ISD_STRICT_FCMP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7894 if (RetVT.SimpleTy != MVT::i32)
7895 return Register();
7896 if ((Subtarget->hasFPARMv8())) {
7897 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
7898 }
7899 return Register();
7900}
7901
7902Register fastEmit_AArch64ISD_STRICT_FCMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7903 switch (VT.SimpleTy) {
7904 case MVT::f16: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
7905 case MVT::f32: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
7906 case MVT::f64: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
7907 default: return Register();
7908 }
7909}
7910
7911// FastEmit functions for AArch64ISD::STRICT_FCMPE.
7912
7913Register fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
7914 if (RetVT.SimpleTy != MVT::i32)
7915 return Register();
7916 if ((Subtarget->hasFullFP16())) {
7917 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPEHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
7918 }
7919 return Register();
7920}
7921
7922Register fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7923 if (RetVT.SimpleTy != MVT::i32)
7924 return Register();
7925 if ((Subtarget->hasFPARMv8())) {
7926 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPESrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
7927 }
7928 return Register();
7929}
7930
7931Register fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7932 if (RetVT.SimpleTy != MVT::i32)
7933 return Register();
7934 if ((Subtarget->hasFPARMv8())) {
7935 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPEDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
7936 }
7937 return Register();
7938}
7939
7940Register fastEmit_AArch64ISD_STRICT_FCMPE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7941 switch (VT.SimpleTy) {
7942 case MVT::f16: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f16_rr(RetVT, Op0, Op1);
7943 case MVT::f32: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f32_rr(RetVT, Op0, Op1);
7944 case MVT::f64: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f64_rr(RetVT, Op0, Op1);
7945 default: return Register();
7946 }
7947}
7948
7949// FastEmit functions for AArch64ISD::TBL.
7950
7951Register fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7952 if (RetVT.SimpleTy != MVT::nxv16i8)
7953 return Register();
7954 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7955 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
7956 }
7957 return Register();
7958}
7959
7960Register fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7961 if (RetVT.SimpleTy != MVT::nxv8i16)
7962 return Register();
7963 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7964 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7965 }
7966 return Register();
7967}
7968
7969Register fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7970 if (RetVT.SimpleTy != MVT::nxv4i32)
7971 return Register();
7972 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7973 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7974 }
7975 return Register();
7976}
7977
7978Register fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7979 if (RetVT.SimpleTy != MVT::nxv2i64)
7980 return Register();
7981 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7982 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7983 }
7984 return Register();
7985}
7986
7987Register fastEmit_AArch64ISD_TBL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7988 switch (VT.SimpleTy) {
7989 case MVT::nxv16i8: return fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(RetVT, Op0, Op1);
7990 case MVT::nxv8i16: return fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(RetVT, Op0, Op1);
7991 case MVT::nxv4i32: return fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(RetVT, Op0, Op1);
7992 case MVT::nxv2i64: return fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(RetVT, Op0, Op1);
7993 default: return Register();
7994 }
7995}
7996
7997// FastEmit functions for AArch64ISD::TRN1.
7998
7999Register fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8000 if (RetVT.SimpleTy != MVT::v8i8)
8001 return Register();
8002 if ((Subtarget->isNeonAvailable())) {
8003 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8004 }
8005 return Register();
8006}
8007
8008Register fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8009 if (RetVT.SimpleTy != MVT::v16i8)
8010 return Register();
8011 if ((Subtarget->isNeonAvailable())) {
8012 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8013 }
8014 return Register();
8015}
8016
8017Register fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8018 if (RetVT.SimpleTy != MVT::v4i16)
8019 return Register();
8020 if ((Subtarget->isNeonAvailable())) {
8021 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8022 }
8023 return Register();
8024}
8025
8026Register fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8027 if (RetVT.SimpleTy != MVT::v8i16)
8028 return Register();
8029 if ((Subtarget->isNeonAvailable())) {
8030 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8031 }
8032 return Register();
8033}
8034
8035Register fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8036 if (RetVT.SimpleTy != MVT::v2i32)
8037 return Register();
8038 if ((Subtarget->isNeonAvailable())) {
8039 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8040 }
8041 return Register();
8042}
8043
8044Register fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8045 if (RetVT.SimpleTy != MVT::v4i32)
8046 return Register();
8047 if ((Subtarget->isNeonAvailable())) {
8048 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8049 }
8050 return Register();
8051}
8052
8053Register fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8054 if (RetVT.SimpleTy != MVT::v2i64)
8055 return Register();
8056 if ((Subtarget->isNeonAvailable())) {
8057 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8058 }
8059 return Register();
8060}
8061
8062Register fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8063 if (RetVT.SimpleTy != MVT::v4f16)
8064 return Register();
8065 if ((Subtarget->isNeonAvailable())) {
8066 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8067 }
8068 return Register();
8069}
8070
8071Register fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8072 if (RetVT.SimpleTy != MVT::v8f16)
8073 return Register();
8074 if ((Subtarget->isNeonAvailable())) {
8075 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8076 }
8077 return Register();
8078}
8079
8080Register fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8081 if (RetVT.SimpleTy != MVT::v4bf16)
8082 return Register();
8083 if ((Subtarget->isNeonAvailable())) {
8084 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8085 }
8086 return Register();
8087}
8088
8089Register fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8090 if (RetVT.SimpleTy != MVT::v8bf16)
8091 return Register();
8092 if ((Subtarget->isNeonAvailable())) {
8093 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8094 }
8095 return Register();
8096}
8097
8098Register fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8099 if (RetVT.SimpleTy != MVT::v2f32)
8100 return Register();
8101 if ((Subtarget->isNeonAvailable())) {
8102 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8103 }
8104 return Register();
8105}
8106
8107Register fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8108 if (RetVT.SimpleTy != MVT::v4f32)
8109 return Register();
8110 if ((Subtarget->isNeonAvailable())) {
8111 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8112 }
8113 return Register();
8114}
8115
8116Register fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8117 if (RetVT.SimpleTy != MVT::v2f64)
8118 return Register();
8119 if ((Subtarget->isNeonAvailable())) {
8120 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8121 }
8122 return Register();
8123}
8124
8125Register fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8126 if (RetVT.SimpleTy != MVT::nxv2i1)
8127 return Register();
8128 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8129 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8130 }
8131 return Register();
8132}
8133
8134Register fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8135 if (RetVT.SimpleTy != MVT::nxv4i1)
8136 return Register();
8137 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8138 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8139 }
8140 return Register();
8141}
8142
8143Register fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8144 if (RetVT.SimpleTy != MVT::nxv8i1)
8145 return Register();
8146 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8147 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8148 }
8149 return Register();
8150}
8151
8152Register fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8153 if (RetVT.SimpleTy != MVT::nxv16i1)
8154 return Register();
8155 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8156 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8157 }
8158 return Register();
8159}
8160
8161Register fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8162 if (RetVT.SimpleTy != MVT::nxv16i8)
8163 return Register();
8164 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8165 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8166 }
8167 return Register();
8168}
8169
8170Register fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8171 if (RetVT.SimpleTy != MVT::nxv8i16)
8172 return Register();
8173 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8174 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8175 }
8176 return Register();
8177}
8178
8179Register fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8180 if (RetVT.SimpleTy != MVT::nxv4i32)
8181 return Register();
8182 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8183 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8184 }
8185 return Register();
8186}
8187
8188Register fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8189 if (RetVT.SimpleTy != MVT::nxv2i64)
8190 return Register();
8191 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8192 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8193 }
8194 return Register();
8195}
8196
8197Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8198 if (RetVT.SimpleTy != MVT::nxv2f16)
8199 return Register();
8200 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8201 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8202 }
8203 return Register();
8204}
8205
8206Register fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8207 if (RetVT.SimpleTy != MVT::nxv4f16)
8208 return Register();
8209 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8210 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8211 }
8212 return Register();
8213}
8214
8215Register fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8216 if (RetVT.SimpleTy != MVT::nxv8f16)
8217 return Register();
8218 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8219 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8220 }
8221 return Register();
8222}
8223
8224Register fastEmit_AArch64ISD_TRN1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8225 if (RetVT.SimpleTy != MVT::nxv2bf16)
8226 return Register();
8227 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8228 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8229 }
8230 return Register();
8231}
8232
8233Register fastEmit_AArch64ISD_TRN1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8234 if (RetVT.SimpleTy != MVT::nxv4bf16)
8235 return Register();
8236 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8237 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8238 }
8239 return Register();
8240}
8241
8242Register fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8243 if (RetVT.SimpleTy != MVT::nxv8bf16)
8244 return Register();
8245 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8246 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8247 }
8248 return Register();
8249}
8250
8251Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8252 if (RetVT.SimpleTy != MVT::nxv2f32)
8253 return Register();
8254 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8255 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8256 }
8257 return Register();
8258}
8259
8260Register fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8261 if (RetVT.SimpleTy != MVT::nxv4f32)
8262 return Register();
8263 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8264 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8265 }
8266 return Register();
8267}
8268
8269Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8270 if (RetVT.SimpleTy != MVT::nxv2f64)
8271 return Register();
8272 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8273 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8274 }
8275 return Register();
8276}
8277
8278Register fastEmit_AArch64ISD_TRN1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8279 switch (VT.SimpleTy) {
8280 case MVT::v8i8: return fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(RetVT, Op0, Op1);
8281 case MVT::v16i8: return fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(RetVT, Op0, Op1);
8282 case MVT::v4i16: return fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(RetVT, Op0, Op1);
8283 case MVT::v8i16: return fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(RetVT, Op0, Op1);
8284 case MVT::v2i32: return fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(RetVT, Op0, Op1);
8285 case MVT::v4i32: return fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(RetVT, Op0, Op1);
8286 case MVT::v2i64: return fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(RetVT, Op0, Op1);
8287 case MVT::v4f16: return fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(RetVT, Op0, Op1);
8288 case MVT::v8f16: return fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(RetVT, Op0, Op1);
8289 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(RetVT, Op0, Op1);
8290 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(RetVT, Op0, Op1);
8291 case MVT::v2f32: return fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(RetVT, Op0, Op1);
8292 case MVT::v4f32: return fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(RetVT, Op0, Op1);
8293 case MVT::v2f64: return fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(RetVT, Op0, Op1);
8294 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8295 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8296 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8297 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8298 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8299 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8300 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8301 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8302 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8303 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8304 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8305 case MVT::nxv2bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8306 case MVT::nxv4bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8307 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8308 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8309 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8310 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8311 default: return Register();
8312 }
8313}
8314
8315// FastEmit functions for AArch64ISD::TRN2.
8316
8317Register fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8318 if (RetVT.SimpleTy != MVT::v8i8)
8319 return Register();
8320 if ((Subtarget->isNeonAvailable())) {
8321 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8322 }
8323 return Register();
8324}
8325
8326Register fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8327 if (RetVT.SimpleTy != MVT::v16i8)
8328 return Register();
8329 if ((Subtarget->isNeonAvailable())) {
8330 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8331 }
8332 return Register();
8333}
8334
8335Register fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8336 if (RetVT.SimpleTy != MVT::v4i16)
8337 return Register();
8338 if ((Subtarget->isNeonAvailable())) {
8339 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8340 }
8341 return Register();
8342}
8343
8344Register fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8345 if (RetVT.SimpleTy != MVT::v8i16)
8346 return Register();
8347 if ((Subtarget->isNeonAvailable())) {
8348 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8349 }
8350 return Register();
8351}
8352
8353Register fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8354 if (RetVT.SimpleTy != MVT::v2i32)
8355 return Register();
8356 if ((Subtarget->isNeonAvailable())) {
8357 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8358 }
8359 return Register();
8360}
8361
8362Register fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8363 if (RetVT.SimpleTy != MVT::v4i32)
8364 return Register();
8365 if ((Subtarget->isNeonAvailable())) {
8366 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8367 }
8368 return Register();
8369}
8370
8371Register fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8372 if (RetVT.SimpleTy != MVT::v2i64)
8373 return Register();
8374 if ((Subtarget->isNeonAvailable())) {
8375 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8376 }
8377 return Register();
8378}
8379
8380Register fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8381 if (RetVT.SimpleTy != MVT::v4f16)
8382 return Register();
8383 if ((Subtarget->isNeonAvailable())) {
8384 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8385 }
8386 return Register();
8387}
8388
8389Register fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8390 if (RetVT.SimpleTy != MVT::v8f16)
8391 return Register();
8392 if ((Subtarget->isNeonAvailable())) {
8393 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8394 }
8395 return Register();
8396}
8397
8398Register fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8399 if (RetVT.SimpleTy != MVT::v4bf16)
8400 return Register();
8401 if ((Subtarget->isNeonAvailable())) {
8402 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8403 }
8404 return Register();
8405}
8406
8407Register fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8408 if (RetVT.SimpleTy != MVT::v8bf16)
8409 return Register();
8410 if ((Subtarget->isNeonAvailable())) {
8411 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8412 }
8413 return Register();
8414}
8415
8416Register fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8417 if (RetVT.SimpleTy != MVT::v2f32)
8418 return Register();
8419 if ((Subtarget->isNeonAvailable())) {
8420 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8421 }
8422 return Register();
8423}
8424
8425Register fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8426 if (RetVT.SimpleTy != MVT::v4f32)
8427 return Register();
8428 if ((Subtarget->isNeonAvailable())) {
8429 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8430 }
8431 return Register();
8432}
8433
8434Register fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8435 if (RetVT.SimpleTy != MVT::v2f64)
8436 return Register();
8437 if ((Subtarget->isNeonAvailable())) {
8438 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8439 }
8440 return Register();
8441}
8442
8443Register fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8444 if (RetVT.SimpleTy != MVT::nxv2i1)
8445 return Register();
8446 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8447 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8448 }
8449 return Register();
8450}
8451
8452Register fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8453 if (RetVT.SimpleTy != MVT::nxv4i1)
8454 return Register();
8455 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8456 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8457 }
8458 return Register();
8459}
8460
8461Register fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8462 if (RetVT.SimpleTy != MVT::nxv8i1)
8463 return Register();
8464 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8465 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8466 }
8467 return Register();
8468}
8469
8470Register fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8471 if (RetVT.SimpleTy != MVT::nxv16i1)
8472 return Register();
8473 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8474 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8475 }
8476 return Register();
8477}
8478
8479Register fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8480 if (RetVT.SimpleTy != MVT::nxv16i8)
8481 return Register();
8482 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8483 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8484 }
8485 return Register();
8486}
8487
8488Register fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8489 if (RetVT.SimpleTy != MVT::nxv8i16)
8490 return Register();
8491 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8492 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8493 }
8494 return Register();
8495}
8496
8497Register fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8498 if (RetVT.SimpleTy != MVT::nxv4i32)
8499 return Register();
8500 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8501 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8502 }
8503 return Register();
8504}
8505
8506Register fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8507 if (RetVT.SimpleTy != MVT::nxv2i64)
8508 return Register();
8509 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8510 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8511 }
8512 return Register();
8513}
8514
8515Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8516 if (RetVT.SimpleTy != MVT::nxv2f16)
8517 return Register();
8518 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8519 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8520 }
8521 return Register();
8522}
8523
8524Register fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8525 if (RetVT.SimpleTy != MVT::nxv4f16)
8526 return Register();
8527 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8528 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8529 }
8530 return Register();
8531}
8532
8533Register fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8534 if (RetVT.SimpleTy != MVT::nxv8f16)
8535 return Register();
8536 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8537 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8538 }
8539 return Register();
8540}
8541
8542Register fastEmit_AArch64ISD_TRN2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8543 if (RetVT.SimpleTy != MVT::nxv2bf16)
8544 return Register();
8545 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8546 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8547 }
8548 return Register();
8549}
8550
8551Register fastEmit_AArch64ISD_TRN2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8552 if (RetVT.SimpleTy != MVT::nxv4bf16)
8553 return Register();
8554 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8555 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8556 }
8557 return Register();
8558}
8559
8560Register fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8561 if (RetVT.SimpleTy != MVT::nxv8bf16)
8562 return Register();
8563 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8564 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8565 }
8566 return Register();
8567}
8568
8569Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8570 if (RetVT.SimpleTy != MVT::nxv2f32)
8571 return Register();
8572 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8573 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8574 }
8575 return Register();
8576}
8577
8578Register fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8579 if (RetVT.SimpleTy != MVT::nxv4f32)
8580 return Register();
8581 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8582 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8583 }
8584 return Register();
8585}
8586
8587Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8588 if (RetVT.SimpleTy != MVT::nxv2f64)
8589 return Register();
8590 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8591 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8592 }
8593 return Register();
8594}
8595
8596Register fastEmit_AArch64ISD_TRN2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8597 switch (VT.SimpleTy) {
8598 case MVT::v8i8: return fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(RetVT, Op0, Op1);
8599 case MVT::v16i8: return fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(RetVT, Op0, Op1);
8600 case MVT::v4i16: return fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(RetVT, Op0, Op1);
8601 case MVT::v8i16: return fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(RetVT, Op0, Op1);
8602 case MVT::v2i32: return fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(RetVT, Op0, Op1);
8603 case MVT::v4i32: return fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(RetVT, Op0, Op1);
8604 case MVT::v2i64: return fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(RetVT, Op0, Op1);
8605 case MVT::v4f16: return fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(RetVT, Op0, Op1);
8606 case MVT::v8f16: return fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(RetVT, Op0, Op1);
8607 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(RetVT, Op0, Op1);
8608 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(RetVT, Op0, Op1);
8609 case MVT::v2f32: return fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(RetVT, Op0, Op1);
8610 case MVT::v4f32: return fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(RetVT, Op0, Op1);
8611 case MVT::v2f64: return fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(RetVT, Op0, Op1);
8612 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8613 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8614 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8615 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8616 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8617 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8618 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8619 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8620 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8621 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8622 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8623 case MVT::nxv2bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8624 case MVT::nxv4bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8625 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8626 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8627 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8628 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8629 default: return Register();
8630 }
8631}
8632
8633// FastEmit functions for AArch64ISD::UMULL.
8634
8635Register fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8636 if (RetVT.SimpleTy != MVT::v8i16)
8637 return Register();
8638 if ((Subtarget->isNeonAvailable())) {
8639 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv8i8_v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8640 }
8641 return Register();
8642}
8643
8644Register fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8645 if (RetVT.SimpleTy != MVT::v4i32)
8646 return Register();
8647 if ((Subtarget->isNeonAvailable())) {
8648 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv4i16_v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8649 }
8650 return Register();
8651}
8652
8653Register fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8654 if (RetVT.SimpleTy != MVT::v2i64)
8655 return Register();
8656 if ((Subtarget->isNeonAvailable())) {
8657 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv2i32_v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8658 }
8659 return Register();
8660}
8661
8662Register fastEmit_AArch64ISD_UMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8663 switch (VT.SimpleTy) {
8664 case MVT::v8i8: return fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
8665 case MVT::v4i16: return fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(RetVT, Op0, Op1);
8666 case MVT::v2i32: return fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(RetVT, Op0, Op1);
8667 default: return Register();
8668 }
8669}
8670
8671// FastEmit functions for AArch64ISD::UQADD.
8672
8673Register fastEmit_AArch64ISD_UQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8674 if (RetVT.SimpleTy != MVT::f32)
8675 return Register();
8676 if ((Subtarget->isNeonAvailable())) {
8677 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8678 }
8679 return Register();
8680}
8681
8682Register fastEmit_AArch64ISD_UQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8683 if (RetVT.SimpleTy != MVT::f64)
8684 return Register();
8685 if ((Subtarget->isNeonAvailable())) {
8686 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8687 }
8688 return Register();
8689}
8690
8691Register fastEmit_AArch64ISD_UQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8692 switch (VT.SimpleTy) {
8693 case MVT::f32: return fastEmit_AArch64ISD_UQADD_MVT_f32_rr(RetVT, Op0, Op1);
8694 case MVT::f64: return fastEmit_AArch64ISD_UQADD_MVT_f64_rr(RetVT, Op0, Op1);
8695 default: return Register();
8696 }
8697}
8698
8699// FastEmit functions for AArch64ISD::UQRSHL.
8700
8701Register fastEmit_AArch64ISD_UQRSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8702 if (RetVT.SimpleTy != MVT::f32)
8703 return Register();
8704 if ((Subtarget->isNeonAvailable())) {
8705 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQRSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8706 }
8707 return Register();
8708}
8709
8710Register fastEmit_AArch64ISD_UQRSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8711 if (RetVT.SimpleTy != MVT::f64)
8712 return Register();
8713 if ((Subtarget->isNeonAvailable())) {
8714 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQRSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8715 }
8716 return Register();
8717}
8718
8719Register fastEmit_AArch64ISD_UQRSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8720 switch (VT.SimpleTy) {
8721 case MVT::f32: return fastEmit_AArch64ISD_UQRSHL_MVT_f32_rr(RetVT, Op0, Op1);
8722 case MVT::f64: return fastEmit_AArch64ISD_UQRSHL_MVT_f64_rr(RetVT, Op0, Op1);
8723 default: return Register();
8724 }
8725}
8726
8727// FastEmit functions for AArch64ISD::UQSHL.
8728
8729Register fastEmit_AArch64ISD_UQSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8730 if (RetVT.SimpleTy != MVT::f32)
8731 return Register();
8732 if ((Subtarget->isNeonAvailable())) {
8733 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8734 }
8735 return Register();
8736}
8737
8738Register fastEmit_AArch64ISD_UQSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8739 if (RetVT.SimpleTy != MVT::f64)
8740 return Register();
8741 if ((Subtarget->isNeonAvailable())) {
8742 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8743 }
8744 return Register();
8745}
8746
8747Register fastEmit_AArch64ISD_UQSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8748 switch (VT.SimpleTy) {
8749 case MVT::f32: return fastEmit_AArch64ISD_UQSHL_MVT_f32_rr(RetVT, Op0, Op1);
8750 case MVT::f64: return fastEmit_AArch64ISD_UQSHL_MVT_f64_rr(RetVT, Op0, Op1);
8751 default: return Register();
8752 }
8753}
8754
8755// FastEmit functions for AArch64ISD::UQSUB.
8756
8757Register fastEmit_AArch64ISD_UQSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8758 if (RetVT.SimpleTy != MVT::f32)
8759 return Register();
8760 if ((Subtarget->isNeonAvailable())) {
8761 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8762 }
8763 return Register();
8764}
8765
8766Register fastEmit_AArch64ISD_UQSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8767 if (RetVT.SimpleTy != MVT::f64)
8768 return Register();
8769 if ((Subtarget->isNeonAvailable())) {
8770 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8771 }
8772 return Register();
8773}
8774
8775Register fastEmit_AArch64ISD_UQSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8776 switch (VT.SimpleTy) {
8777 case MVT::f32: return fastEmit_AArch64ISD_UQSUB_MVT_f32_rr(RetVT, Op0, Op1);
8778 case MVT::f64: return fastEmit_AArch64ISD_UQSUB_MVT_f64_rr(RetVT, Op0, Op1);
8779 default: return Register();
8780 }
8781}
8782
8783// FastEmit functions for AArch64ISD::UZP1.
8784
8785Register fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8786 if (RetVT.SimpleTy != MVT::v8i8)
8787 return Register();
8788 if ((Subtarget->isNeonAvailable())) {
8789 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8790 }
8791 return Register();
8792}
8793
8794Register fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8795 if (RetVT.SimpleTy != MVT::v16i8)
8796 return Register();
8797 if ((Subtarget->isNeonAvailable())) {
8798 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8799 }
8800 return Register();
8801}
8802
8803Register fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8804 if (RetVT.SimpleTy != MVT::v4i16)
8805 return Register();
8806 if ((Subtarget->isNeonAvailable())) {
8807 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8808 }
8809 return Register();
8810}
8811
8812Register fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8813 if (RetVT.SimpleTy != MVT::v8i16)
8814 return Register();
8815 if ((Subtarget->isNeonAvailable())) {
8816 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8817 }
8818 return Register();
8819}
8820
8821Register fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8822 if (RetVT.SimpleTy != MVT::v2i32)
8823 return Register();
8824 if ((Subtarget->isNeonAvailable())) {
8825 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8826 }
8827 return Register();
8828}
8829
8830Register fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8831 if (RetVT.SimpleTy != MVT::v4i32)
8832 return Register();
8833 if ((Subtarget->isNeonAvailable())) {
8834 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8835 }
8836 return Register();
8837}
8838
8839Register fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8840 if (RetVT.SimpleTy != MVT::v2i64)
8841 return Register();
8842 if ((Subtarget->isNeonAvailable())) {
8843 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8844 }
8845 return Register();
8846}
8847
8848Register fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8849 if (RetVT.SimpleTy != MVT::v4f16)
8850 return Register();
8851 if ((Subtarget->isNeonAvailable())) {
8852 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8853 }
8854 return Register();
8855}
8856
8857Register fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8858 if (RetVT.SimpleTy != MVT::v8f16)
8859 return Register();
8860 if ((Subtarget->isNeonAvailable())) {
8861 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8862 }
8863 return Register();
8864}
8865
8866Register fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8867 if (RetVT.SimpleTy != MVT::v4bf16)
8868 return Register();
8869 if ((Subtarget->isNeonAvailable())) {
8870 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8871 }
8872 return Register();
8873}
8874
8875Register fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8876 if (RetVT.SimpleTy != MVT::v8bf16)
8877 return Register();
8878 if ((Subtarget->isNeonAvailable())) {
8879 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8880 }
8881 return Register();
8882}
8883
8884Register fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8885 if (RetVT.SimpleTy != MVT::v2f32)
8886 return Register();
8887 if ((Subtarget->isNeonAvailable())) {
8888 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8889 }
8890 return Register();
8891}
8892
8893Register fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8894 if (RetVT.SimpleTy != MVT::v4f32)
8895 return Register();
8896 if ((Subtarget->isNeonAvailable())) {
8897 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8898 }
8899 return Register();
8900}
8901
8902Register fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8903 if (RetVT.SimpleTy != MVT::v2f64)
8904 return Register();
8905 if ((Subtarget->isNeonAvailable())) {
8906 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8907 }
8908 return Register();
8909}
8910
8911Register fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8912 if (RetVT.SimpleTy != MVT::nxv2i1)
8913 return Register();
8914 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8915 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8916 }
8917 return Register();
8918}
8919
8920Register fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8921 if (RetVT.SimpleTy != MVT::nxv4i1)
8922 return Register();
8923 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8924 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8925 }
8926 return Register();
8927}
8928
8929Register fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8930 if (RetVT.SimpleTy != MVT::nxv8i1)
8931 return Register();
8932 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8933 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8934 }
8935 return Register();
8936}
8937
8938Register fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8939 if (RetVT.SimpleTy != MVT::nxv16i1)
8940 return Register();
8941 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8942 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8943 }
8944 return Register();
8945}
8946
8947Register fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8948 if (RetVT.SimpleTy != MVT::nxv16i8)
8949 return Register();
8950 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8951 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8952 }
8953 return Register();
8954}
8955
8956Register fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8957 if (RetVT.SimpleTy != MVT::nxv8i16)
8958 return Register();
8959 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8960 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8961 }
8962 return Register();
8963}
8964
8965Register fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8966 if (RetVT.SimpleTy != MVT::nxv4i32)
8967 return Register();
8968 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8969 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8970 }
8971 return Register();
8972}
8973
8974Register fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8975 if (RetVT.SimpleTy != MVT::nxv2i64)
8976 return Register();
8977 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8978 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8979 }
8980 return Register();
8981}
8982
8983Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8984 if (RetVT.SimpleTy != MVT::nxv2f16)
8985 return Register();
8986 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8987 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8988 }
8989 return Register();
8990}
8991
8992Register fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8993 if (RetVT.SimpleTy != MVT::nxv4f16)
8994 return Register();
8995 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8996 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8997 }
8998 return Register();
8999}
9000
9001Register fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9002 if (RetVT.SimpleTy != MVT::nxv8f16)
9003 return Register();
9004 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9005 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9006 }
9007 return Register();
9008}
9009
9010Register fastEmit_AArch64ISD_UZP1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9011 if (RetVT.SimpleTy != MVT::nxv2bf16)
9012 return Register();
9013 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9014 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9015 }
9016 return Register();
9017}
9018
9019Register fastEmit_AArch64ISD_UZP1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9020 if (RetVT.SimpleTy != MVT::nxv4bf16)
9021 return Register();
9022 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9023 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9024 }
9025 return Register();
9026}
9027
9028Register fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9029 if (RetVT.SimpleTy != MVT::nxv8bf16)
9030 return Register();
9031 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9032 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9033 }
9034 return Register();
9035}
9036
9037Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9038 if (RetVT.SimpleTy != MVT::nxv2f32)
9039 return Register();
9040 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9041 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9042 }
9043 return Register();
9044}
9045
9046Register fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9047 if (RetVT.SimpleTy != MVT::nxv4f32)
9048 return Register();
9049 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9050 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9051 }
9052 return Register();
9053}
9054
9055Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9056 if (RetVT.SimpleTy != MVT::nxv2f64)
9057 return Register();
9058 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9059 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9060 }
9061 return Register();
9062}
9063
9064Register fastEmit_AArch64ISD_UZP1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9065 switch (VT.SimpleTy) {
9066 case MVT::v8i8: return fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(RetVT, Op0, Op1);
9067 case MVT::v16i8: return fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(RetVT, Op0, Op1);
9068 case MVT::v4i16: return fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(RetVT, Op0, Op1);
9069 case MVT::v8i16: return fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(RetVT, Op0, Op1);
9070 case MVT::v2i32: return fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(RetVT, Op0, Op1);
9071 case MVT::v4i32: return fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(RetVT, Op0, Op1);
9072 case MVT::v2i64: return fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(RetVT, Op0, Op1);
9073 case MVT::v4f16: return fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(RetVT, Op0, Op1);
9074 case MVT::v8f16: return fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(RetVT, Op0, Op1);
9075 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(RetVT, Op0, Op1);
9076 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(RetVT, Op0, Op1);
9077 case MVT::v2f32: return fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(RetVT, Op0, Op1);
9078 case MVT::v4f32: return fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(RetVT, Op0, Op1);
9079 case MVT::v2f64: return fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(RetVT, Op0, Op1);
9080 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9081 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9082 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9083 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9084 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9085 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9086 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9087 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9088 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9089 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9090 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9091 case MVT::nxv2bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9092 case MVT::nxv4bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9093 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9094 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9095 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9096 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9097 default: return Register();
9098 }
9099}
9100
9101// FastEmit functions for AArch64ISD::UZP2.
9102
9103Register fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9104 if (RetVT.SimpleTy != MVT::v8i8)
9105 return Register();
9106 if ((Subtarget->isNeonAvailable())) {
9107 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9108 }
9109 return Register();
9110}
9111
9112Register fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9113 if (RetVT.SimpleTy != MVT::v16i8)
9114 return Register();
9115 if ((Subtarget->isNeonAvailable())) {
9116 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9117 }
9118 return Register();
9119}
9120
9121Register fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9122 if (RetVT.SimpleTy != MVT::v4i16)
9123 return Register();
9124 if ((Subtarget->isNeonAvailable())) {
9125 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9126 }
9127 return Register();
9128}
9129
9130Register fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9131 if (RetVT.SimpleTy != MVT::v8i16)
9132 return Register();
9133 if ((Subtarget->isNeonAvailable())) {
9134 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9135 }
9136 return Register();
9137}
9138
9139Register fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9140 if (RetVT.SimpleTy != MVT::v2i32)
9141 return Register();
9142 if ((Subtarget->isNeonAvailable())) {
9143 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9144 }
9145 return Register();
9146}
9147
9148Register fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9149 if (RetVT.SimpleTy != MVT::v4i32)
9150 return Register();
9151 if ((Subtarget->isNeonAvailable())) {
9152 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9153 }
9154 return Register();
9155}
9156
9157Register fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9158 if (RetVT.SimpleTy != MVT::v2i64)
9159 return Register();
9160 if ((Subtarget->isNeonAvailable())) {
9161 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9162 }
9163 return Register();
9164}
9165
9166Register fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9167 if (RetVT.SimpleTy != MVT::v4f16)
9168 return Register();
9169 if ((Subtarget->isNeonAvailable())) {
9170 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9171 }
9172 return Register();
9173}
9174
9175Register fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9176 if (RetVT.SimpleTy != MVT::v8f16)
9177 return Register();
9178 if ((Subtarget->isNeonAvailable())) {
9179 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9180 }
9181 return Register();
9182}
9183
9184Register fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9185 if (RetVT.SimpleTy != MVT::v4bf16)
9186 return Register();
9187 if ((Subtarget->isNeonAvailable())) {
9188 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9189 }
9190 return Register();
9191}
9192
9193Register fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9194 if (RetVT.SimpleTy != MVT::v8bf16)
9195 return Register();
9196 if ((Subtarget->isNeonAvailable())) {
9197 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9198 }
9199 return Register();
9200}
9201
9202Register fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9203 if (RetVT.SimpleTy != MVT::v2f32)
9204 return Register();
9205 if ((Subtarget->isNeonAvailable())) {
9206 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9207 }
9208 return Register();
9209}
9210
9211Register fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9212 if (RetVT.SimpleTy != MVT::v4f32)
9213 return Register();
9214 if ((Subtarget->isNeonAvailable())) {
9215 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9216 }
9217 return Register();
9218}
9219
9220Register fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9221 if (RetVT.SimpleTy != MVT::v2f64)
9222 return Register();
9223 if ((Subtarget->isNeonAvailable())) {
9224 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9225 }
9226 return Register();
9227}
9228
9229Register fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9230 if (RetVT.SimpleTy != MVT::nxv2i1)
9231 return Register();
9232 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9233 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9234 }
9235 return Register();
9236}
9237
9238Register fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9239 if (RetVT.SimpleTy != MVT::nxv4i1)
9240 return Register();
9241 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9242 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9243 }
9244 return Register();
9245}
9246
9247Register fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9248 if (RetVT.SimpleTy != MVT::nxv8i1)
9249 return Register();
9250 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9251 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9252 }
9253 return Register();
9254}
9255
9256Register fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9257 if (RetVT.SimpleTy != MVT::nxv16i1)
9258 return Register();
9259 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9260 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9261 }
9262 return Register();
9263}
9264
9265Register fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9266 if (RetVT.SimpleTy != MVT::nxv16i8)
9267 return Register();
9268 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9269 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9270 }
9271 return Register();
9272}
9273
9274Register fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9275 if (RetVT.SimpleTy != MVT::nxv8i16)
9276 return Register();
9277 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9278 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9279 }
9280 return Register();
9281}
9282
9283Register fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9284 if (RetVT.SimpleTy != MVT::nxv4i32)
9285 return Register();
9286 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9287 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9288 }
9289 return Register();
9290}
9291
9292Register fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9293 if (RetVT.SimpleTy != MVT::nxv2i64)
9294 return Register();
9295 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9296 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9297 }
9298 return Register();
9299}
9300
9301Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9302 if (RetVT.SimpleTy != MVT::nxv2f16)
9303 return Register();
9304 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9305 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9306 }
9307 return Register();
9308}
9309
9310Register fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9311 if (RetVT.SimpleTy != MVT::nxv4f16)
9312 return Register();
9313 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9314 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9315 }
9316 return Register();
9317}
9318
9319Register fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9320 if (RetVT.SimpleTy != MVT::nxv8f16)
9321 return Register();
9322 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9323 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9324 }
9325 return Register();
9326}
9327
9328Register fastEmit_AArch64ISD_UZP2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9329 if (RetVT.SimpleTy != MVT::nxv2bf16)
9330 return Register();
9331 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9332 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9333 }
9334 return Register();
9335}
9336
9337Register fastEmit_AArch64ISD_UZP2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9338 if (RetVT.SimpleTy != MVT::nxv4bf16)
9339 return Register();
9340 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9341 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9342 }
9343 return Register();
9344}
9345
9346Register fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9347 if (RetVT.SimpleTy != MVT::nxv8bf16)
9348 return Register();
9349 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9350 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9351 }
9352 return Register();
9353}
9354
9355Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9356 if (RetVT.SimpleTy != MVT::nxv2f32)
9357 return Register();
9358 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9359 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9360 }
9361 return Register();
9362}
9363
9364Register fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9365 if (RetVT.SimpleTy != MVT::nxv4f32)
9366 return Register();
9367 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9368 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9369 }
9370 return Register();
9371}
9372
9373Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9374 if (RetVT.SimpleTy != MVT::nxv2f64)
9375 return Register();
9376 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9377 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9378 }
9379 return Register();
9380}
9381
9382Register fastEmit_AArch64ISD_UZP2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9383 switch (VT.SimpleTy) {
9384 case MVT::v8i8: return fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(RetVT, Op0, Op1);
9385 case MVT::v16i8: return fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(RetVT, Op0, Op1);
9386 case MVT::v4i16: return fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(RetVT, Op0, Op1);
9387 case MVT::v8i16: return fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(RetVT, Op0, Op1);
9388 case MVT::v2i32: return fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(RetVT, Op0, Op1);
9389 case MVT::v4i32: return fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(RetVT, Op0, Op1);
9390 case MVT::v2i64: return fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(RetVT, Op0, Op1);
9391 case MVT::v4f16: return fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(RetVT, Op0, Op1);
9392 case MVT::v8f16: return fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(RetVT, Op0, Op1);
9393 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(RetVT, Op0, Op1);
9394 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(RetVT, Op0, Op1);
9395 case MVT::v2f32: return fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(RetVT, Op0, Op1);
9396 case MVT::v4f32: return fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(RetVT, Op0, Op1);
9397 case MVT::v2f64: return fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(RetVT, Op0, Op1);
9398 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9399 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9400 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9401 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9402 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9403 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9404 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9405 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9406 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9407 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9408 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9409 case MVT::nxv2bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9410 case MVT::nxv4bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9411 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9412 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9413 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9414 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9415 default: return Register();
9416 }
9417}
9418
9419// FastEmit functions for AArch64ISD::ZIP1.
9420
9421Register fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9422 if (RetVT.SimpleTy != MVT::v8i8)
9423 return Register();
9424 if ((Subtarget->isNeonAvailable())) {
9425 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9426 }
9427 return Register();
9428}
9429
9430Register fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9431 if (RetVT.SimpleTy != MVT::v16i8)
9432 return Register();
9433 if ((Subtarget->isNeonAvailable())) {
9434 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9435 }
9436 return Register();
9437}
9438
9439Register fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9440 if (RetVT.SimpleTy != MVT::v4i16)
9441 return Register();
9442 if ((Subtarget->isNeonAvailable())) {
9443 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9444 }
9445 return Register();
9446}
9447
9448Register fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9449 if (RetVT.SimpleTy != MVT::v8i16)
9450 return Register();
9451 if ((Subtarget->isNeonAvailable())) {
9452 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9453 }
9454 return Register();
9455}
9456
9457Register fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9458 if (RetVT.SimpleTy != MVT::v2i32)
9459 return Register();
9460 if ((Subtarget->isNeonAvailable())) {
9461 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9462 }
9463 return Register();
9464}
9465
9466Register fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9467 if (RetVT.SimpleTy != MVT::v4i32)
9468 return Register();
9469 if ((Subtarget->isNeonAvailable())) {
9470 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9471 }
9472 return Register();
9473}
9474
9475Register fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9476 if (RetVT.SimpleTy != MVT::v2i64)
9477 return Register();
9478 if ((Subtarget->isNeonAvailable())) {
9479 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9480 }
9481 return Register();
9482}
9483
9484Register fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9485 if (RetVT.SimpleTy != MVT::v4f16)
9486 return Register();
9487 if ((Subtarget->isNeonAvailable())) {
9488 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9489 }
9490 return Register();
9491}
9492
9493Register fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9494 if (RetVT.SimpleTy != MVT::v8f16)
9495 return Register();
9496 if ((Subtarget->isNeonAvailable())) {
9497 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9498 }
9499 return Register();
9500}
9501
9502Register fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9503 if (RetVT.SimpleTy != MVT::v4bf16)
9504 return Register();
9505 if ((Subtarget->isNeonAvailable())) {
9506 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9507 }
9508 return Register();
9509}
9510
9511Register fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9512 if (RetVT.SimpleTy != MVT::v8bf16)
9513 return Register();
9514 if ((Subtarget->isNeonAvailable())) {
9515 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9516 }
9517 return Register();
9518}
9519
9520Register fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9521 if (RetVT.SimpleTy != MVT::v2f32)
9522 return Register();
9523 if ((Subtarget->isNeonAvailable())) {
9524 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9525 }
9526 return Register();
9527}
9528
9529Register fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9530 if (RetVT.SimpleTy != MVT::v4f32)
9531 return Register();
9532 if ((Subtarget->isNeonAvailable())) {
9533 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9534 }
9535 return Register();
9536}
9537
9538Register fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9539 if (RetVT.SimpleTy != MVT::v2f64)
9540 return Register();
9541 if ((Subtarget->isNeonAvailable())) {
9542 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9543 }
9544 return Register();
9545}
9546
9547Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9548 if (RetVT.SimpleTy != MVT::nxv2i1)
9549 return Register();
9550 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9551 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9552 }
9553 return Register();
9554}
9555
9556Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9557 if (RetVT.SimpleTy != MVT::nxv4i1)
9558 return Register();
9559 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9560 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9561 }
9562 return Register();
9563}
9564
9565Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9566 if (RetVT.SimpleTy != MVT::nxv8i1)
9567 return Register();
9568 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9569 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9570 }
9571 return Register();
9572}
9573
9574Register fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9575 if (RetVT.SimpleTy != MVT::nxv16i1)
9576 return Register();
9577 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9578 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9579 }
9580 return Register();
9581}
9582
9583Register fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9584 if (RetVT.SimpleTy != MVT::nxv16i8)
9585 return Register();
9586 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9587 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9588 }
9589 return Register();
9590}
9591
9592Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9593 if (RetVT.SimpleTy != MVT::nxv8i16)
9594 return Register();
9595 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9596 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9597 }
9598 return Register();
9599}
9600
9601Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9602 if (RetVT.SimpleTy != MVT::nxv4i32)
9603 return Register();
9604 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9605 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9606 }
9607 return Register();
9608}
9609
9610Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9611 if (RetVT.SimpleTy != MVT::nxv2i64)
9612 return Register();
9613 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9614 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9615 }
9616 return Register();
9617}
9618
9619Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9620 if (RetVT.SimpleTy != MVT::nxv2f16)
9621 return Register();
9622 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9623 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9624 }
9625 return Register();
9626}
9627
9628Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9629 if (RetVT.SimpleTy != MVT::nxv4f16)
9630 return Register();
9631 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9632 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9633 }
9634 return Register();
9635}
9636
9637Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9638 if (RetVT.SimpleTy != MVT::nxv8f16)
9639 return Register();
9640 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9641 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9642 }
9643 return Register();
9644}
9645
9646Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9647 if (RetVT.SimpleTy != MVT::nxv2bf16)
9648 return Register();
9649 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9650 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9651 }
9652 return Register();
9653}
9654
9655Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9656 if (RetVT.SimpleTy != MVT::nxv4bf16)
9657 return Register();
9658 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9659 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9660 }
9661 return Register();
9662}
9663
9664Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9665 if (RetVT.SimpleTy != MVT::nxv8bf16)
9666 return Register();
9667 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9668 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9669 }
9670 return Register();
9671}
9672
9673Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9674 if (RetVT.SimpleTy != MVT::nxv2f32)
9675 return Register();
9676 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9677 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9678 }
9679 return Register();
9680}
9681
9682Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9683 if (RetVT.SimpleTy != MVT::nxv4f32)
9684 return Register();
9685 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9686 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9687 }
9688 return Register();
9689}
9690
9691Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9692 if (RetVT.SimpleTy != MVT::nxv2f64)
9693 return Register();
9694 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9695 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9696 }
9697 return Register();
9698}
9699
9700Register fastEmit_AArch64ISD_ZIP1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9701 switch (VT.SimpleTy) {
9702 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(RetVT, Op0, Op1);
9703 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(RetVT, Op0, Op1);
9704 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(RetVT, Op0, Op1);
9705 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(RetVT, Op0, Op1);
9706 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(RetVT, Op0, Op1);
9707 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(RetVT, Op0, Op1);
9708 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(RetVT, Op0, Op1);
9709 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(RetVT, Op0, Op1);
9710 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(RetVT, Op0, Op1);
9711 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(RetVT, Op0, Op1);
9712 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(RetVT, Op0, Op1);
9713 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(RetVT, Op0, Op1);
9714 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(RetVT, Op0, Op1);
9715 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(RetVT, Op0, Op1);
9716 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9717 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9718 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9719 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9720 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9721 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9722 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9723 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9724 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9725 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9726 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9727 case MVT::nxv2bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9728 case MVT::nxv4bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9729 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9730 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9731 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9732 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9733 default: return Register();
9734 }
9735}
9736
9737// FastEmit functions for AArch64ISD::ZIP2.
9738
9739Register fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9740 if (RetVT.SimpleTy != MVT::v8i8)
9741 return Register();
9742 if ((Subtarget->isNeonAvailable())) {
9743 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9744 }
9745 return Register();
9746}
9747
9748Register fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9749 if (RetVT.SimpleTy != MVT::v16i8)
9750 return Register();
9751 if ((Subtarget->isNeonAvailable())) {
9752 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9753 }
9754 return Register();
9755}
9756
9757Register fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9758 if (RetVT.SimpleTy != MVT::v4i16)
9759 return Register();
9760 if ((Subtarget->isNeonAvailable())) {
9761 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9762 }
9763 return Register();
9764}
9765
9766Register fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9767 if (RetVT.SimpleTy != MVT::v8i16)
9768 return Register();
9769 if ((Subtarget->isNeonAvailable())) {
9770 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9771 }
9772 return Register();
9773}
9774
9775Register fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9776 if (RetVT.SimpleTy != MVT::v2i32)
9777 return Register();
9778 if ((Subtarget->isNeonAvailable())) {
9779 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9780 }
9781 return Register();
9782}
9783
9784Register fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9785 if (RetVT.SimpleTy != MVT::v4i32)
9786 return Register();
9787 if ((Subtarget->isNeonAvailable())) {
9788 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9789 }
9790 return Register();
9791}
9792
9793Register fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9794 if (RetVT.SimpleTy != MVT::v2i64)
9795 return Register();
9796 if ((Subtarget->isNeonAvailable())) {
9797 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9798 }
9799 return Register();
9800}
9801
9802Register fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9803 if (RetVT.SimpleTy != MVT::v4f16)
9804 return Register();
9805 if ((Subtarget->isNeonAvailable())) {
9806 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9807 }
9808 return Register();
9809}
9810
9811Register fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9812 if (RetVT.SimpleTy != MVT::v8f16)
9813 return Register();
9814 if ((Subtarget->isNeonAvailable())) {
9815 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9816 }
9817 return Register();
9818}
9819
9820Register fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9821 if (RetVT.SimpleTy != MVT::v4bf16)
9822 return Register();
9823 if ((Subtarget->isNeonAvailable())) {
9824 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9825 }
9826 return Register();
9827}
9828
9829Register fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9830 if (RetVT.SimpleTy != MVT::v8bf16)
9831 return Register();
9832 if ((Subtarget->isNeonAvailable())) {
9833 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9834 }
9835 return Register();
9836}
9837
9838Register fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9839 if (RetVT.SimpleTy != MVT::v2f32)
9840 return Register();
9841 if ((Subtarget->isNeonAvailable())) {
9842 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9843 }
9844 return Register();
9845}
9846
9847Register fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9848 if (RetVT.SimpleTy != MVT::v4f32)
9849 return Register();
9850 if ((Subtarget->isNeonAvailable())) {
9851 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9852 }
9853 return Register();
9854}
9855
9856Register fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9857 if (RetVT.SimpleTy != MVT::v2f64)
9858 return Register();
9859 if ((Subtarget->isNeonAvailable())) {
9860 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9861 }
9862 return Register();
9863}
9864
9865Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9866 if (RetVT.SimpleTy != MVT::nxv2i1)
9867 return Register();
9868 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9869 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9870 }
9871 return Register();
9872}
9873
9874Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9875 if (RetVT.SimpleTy != MVT::nxv4i1)
9876 return Register();
9877 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9878 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9879 }
9880 return Register();
9881}
9882
9883Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9884 if (RetVT.SimpleTy != MVT::nxv8i1)
9885 return Register();
9886 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9887 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9888 }
9889 return Register();
9890}
9891
9892Register fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9893 if (RetVT.SimpleTy != MVT::nxv16i1)
9894 return Register();
9895 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9896 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9897 }
9898 return Register();
9899}
9900
9901Register fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9902 if (RetVT.SimpleTy != MVT::nxv16i8)
9903 return Register();
9904 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9905 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9906 }
9907 return Register();
9908}
9909
9910Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9911 if (RetVT.SimpleTy != MVT::nxv8i16)
9912 return Register();
9913 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9914 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9915 }
9916 return Register();
9917}
9918
9919Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9920 if (RetVT.SimpleTy != MVT::nxv4i32)
9921 return Register();
9922 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9923 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9924 }
9925 return Register();
9926}
9927
9928Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9929 if (RetVT.SimpleTy != MVT::nxv2i64)
9930 return Register();
9931 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9932 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9933 }
9934 return Register();
9935}
9936
9937Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9938 if (RetVT.SimpleTy != MVT::nxv2f16)
9939 return Register();
9940 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9941 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9942 }
9943 return Register();
9944}
9945
9946Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9947 if (RetVT.SimpleTy != MVT::nxv4f16)
9948 return Register();
9949 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9950 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9951 }
9952 return Register();
9953}
9954
9955Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9956 if (RetVT.SimpleTy != MVT::nxv8f16)
9957 return Register();
9958 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9959 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9960 }
9961 return Register();
9962}
9963
9964Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9965 if (RetVT.SimpleTy != MVT::nxv2bf16)
9966 return Register();
9967 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9968 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9969 }
9970 return Register();
9971}
9972
9973Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9974 if (RetVT.SimpleTy != MVT::nxv4bf16)
9975 return Register();
9976 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9977 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9978 }
9979 return Register();
9980}
9981
9982Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9983 if (RetVT.SimpleTy != MVT::nxv8bf16)
9984 return Register();
9985 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9986 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9987 }
9988 return Register();
9989}
9990
9991Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9992 if (RetVT.SimpleTy != MVT::nxv2f32)
9993 return Register();
9994 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9995 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9996 }
9997 return Register();
9998}
9999
10000Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10001 if (RetVT.SimpleTy != MVT::nxv4f32)
10002 return Register();
10003 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10004 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10005 }
10006 return Register();
10007}
10008
10009Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10010 if (RetVT.SimpleTy != MVT::nxv2f64)
10011 return Register();
10012 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10013 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10014 }
10015 return Register();
10016}
10017
10018Register fastEmit_AArch64ISD_ZIP2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10019 switch (VT.SimpleTy) {
10020 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(RetVT, Op0, Op1);
10021 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(RetVT, Op0, Op1);
10022 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(RetVT, Op0, Op1);
10023 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(RetVT, Op0, Op1);
10024 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(RetVT, Op0, Op1);
10025 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(RetVT, Op0, Op1);
10026 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(RetVT, Op0, Op1);
10027 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(RetVT, Op0, Op1);
10028 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(RetVT, Op0, Op1);
10029 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(RetVT, Op0, Op1);
10030 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(RetVT, Op0, Op1);
10031 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(RetVT, Op0, Op1);
10032 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(RetVT, Op0, Op1);
10033 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(RetVT, Op0, Op1);
10034 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
10035 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
10036 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
10037 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
10038 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10039 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10040 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10041 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10042 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
10043 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
10044 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
10045 case MVT::nxv2bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
10046 case MVT::nxv4bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
10047 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
10048 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
10049 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
10050 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
10051 default: return Register();
10052 }
10053}
10054
10055// FastEmit functions for ISD::ABDS.
10056
10057Register fastEmit_ISD_ABDS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10058 if (RetVT.SimpleTy != MVT::v8i8)
10059 return Register();
10060 if ((Subtarget->isNeonAvailable())) {
10061 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10062 }
10063 return Register();
10064}
10065
10066Register fastEmit_ISD_ABDS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10067 if (RetVT.SimpleTy != MVT::v16i8)
10068 return Register();
10069 if ((Subtarget->isNeonAvailable())) {
10070 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10071 }
10072 return Register();
10073}
10074
10075Register fastEmit_ISD_ABDS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10076 if (RetVT.SimpleTy != MVT::v4i16)
10077 return Register();
10078 if ((Subtarget->isNeonAvailable())) {
10079 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10080 }
10081 return Register();
10082}
10083
10084Register fastEmit_ISD_ABDS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10085 if (RetVT.SimpleTy != MVT::v8i16)
10086 return Register();
10087 if ((Subtarget->isNeonAvailable())) {
10088 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10089 }
10090 return Register();
10091}
10092
10093Register fastEmit_ISD_ABDS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10094 if (RetVT.SimpleTy != MVT::v2i32)
10095 return Register();
10096 if ((Subtarget->isNeonAvailable())) {
10097 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10098 }
10099 return Register();
10100}
10101
10102Register fastEmit_ISD_ABDS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10103 if (RetVT.SimpleTy != MVT::v4i32)
10104 return Register();
10105 if ((Subtarget->isNeonAvailable())) {
10106 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10107 }
10108 return Register();
10109}
10110
10111Register fastEmit_ISD_ABDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10112 switch (VT.SimpleTy) {
10113 case MVT::v8i8: return fastEmit_ISD_ABDS_MVT_v8i8_rr(RetVT, Op0, Op1);
10114 case MVT::v16i8: return fastEmit_ISD_ABDS_MVT_v16i8_rr(RetVT, Op0, Op1);
10115 case MVT::v4i16: return fastEmit_ISD_ABDS_MVT_v4i16_rr(RetVT, Op0, Op1);
10116 case MVT::v8i16: return fastEmit_ISD_ABDS_MVT_v8i16_rr(RetVT, Op0, Op1);
10117 case MVT::v2i32: return fastEmit_ISD_ABDS_MVT_v2i32_rr(RetVT, Op0, Op1);
10118 case MVT::v4i32: return fastEmit_ISD_ABDS_MVT_v4i32_rr(RetVT, Op0, Op1);
10119 default: return Register();
10120 }
10121}
10122
10123// FastEmit functions for ISD::ABDU.
10124
10125Register fastEmit_ISD_ABDU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10126 if (RetVT.SimpleTy != MVT::v8i8)
10127 return Register();
10128 if ((Subtarget->isNeonAvailable())) {
10129 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10130 }
10131 return Register();
10132}
10133
10134Register fastEmit_ISD_ABDU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10135 if (RetVT.SimpleTy != MVT::v16i8)
10136 return Register();
10137 if ((Subtarget->isNeonAvailable())) {
10138 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10139 }
10140 return Register();
10141}
10142
10143Register fastEmit_ISD_ABDU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10144 if (RetVT.SimpleTy != MVT::v4i16)
10145 return Register();
10146 if ((Subtarget->isNeonAvailable())) {
10147 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10148 }
10149 return Register();
10150}
10151
10152Register fastEmit_ISD_ABDU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10153 if (RetVT.SimpleTy != MVT::v8i16)
10154 return Register();
10155 if ((Subtarget->isNeonAvailable())) {
10156 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10157 }
10158 return Register();
10159}
10160
10161Register fastEmit_ISD_ABDU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10162 if (RetVT.SimpleTy != MVT::v2i32)
10163 return Register();
10164 if ((Subtarget->isNeonAvailable())) {
10165 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10166 }
10167 return Register();
10168}
10169
10170Register fastEmit_ISD_ABDU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10171 if (RetVT.SimpleTy != MVT::v4i32)
10172 return Register();
10173 if ((Subtarget->isNeonAvailable())) {
10174 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10175 }
10176 return Register();
10177}
10178
10179Register fastEmit_ISD_ABDU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10180 switch (VT.SimpleTy) {
10181 case MVT::v8i8: return fastEmit_ISD_ABDU_MVT_v8i8_rr(RetVT, Op0, Op1);
10182 case MVT::v16i8: return fastEmit_ISD_ABDU_MVT_v16i8_rr(RetVT, Op0, Op1);
10183 case MVT::v4i16: return fastEmit_ISD_ABDU_MVT_v4i16_rr(RetVT, Op0, Op1);
10184 case MVT::v8i16: return fastEmit_ISD_ABDU_MVT_v8i16_rr(RetVT, Op0, Op1);
10185 case MVT::v2i32: return fastEmit_ISD_ABDU_MVT_v2i32_rr(RetVT, Op0, Op1);
10186 case MVT::v4i32: return fastEmit_ISD_ABDU_MVT_v4i32_rr(RetVT, Op0, Op1);
10187 default: return Register();
10188 }
10189}
10190
10191// FastEmit functions for ISD::ADD.
10192
10193Register fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10194 if (RetVT.SimpleTy != MVT::i32)
10195 return Register();
10196 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
10197}
10198
10199Register fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10200 if (RetVT.SimpleTy != MVT::i64)
10201 return Register();
10202 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
10203}
10204
10205Register fastEmit_ISD_ADD_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10206 if (RetVT.SimpleTy != MVT::v8i8)
10207 return Register();
10208 if ((Subtarget->isNeonAvailable())) {
10209 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10210 }
10211 return Register();
10212}
10213
10214Register fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10215 if (RetVT.SimpleTy != MVT::v16i8)
10216 return Register();
10217 if ((Subtarget->isNeonAvailable())) {
10218 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10219 }
10220 return Register();
10221}
10222
10223Register fastEmit_ISD_ADD_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10224 if (RetVT.SimpleTy != MVT::v4i16)
10225 return Register();
10226 if ((Subtarget->isNeonAvailable())) {
10227 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10228 }
10229 return Register();
10230}
10231
10232Register fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10233 if (RetVT.SimpleTy != MVT::v8i16)
10234 return Register();
10235 if ((Subtarget->isNeonAvailable())) {
10236 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10237 }
10238 return Register();
10239}
10240
10241Register fastEmit_ISD_ADD_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10242 if (RetVT.SimpleTy != MVT::v2i32)
10243 return Register();
10244 if ((Subtarget->isNeonAvailable())) {
10245 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10246 }
10247 return Register();
10248}
10249
10250Register fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10251 if (RetVT.SimpleTy != MVT::v4i32)
10252 return Register();
10253 if ((Subtarget->isNeonAvailable())) {
10254 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10255 }
10256 return Register();
10257}
10258
10259Register fastEmit_ISD_ADD_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
10260 if (RetVT.SimpleTy != MVT::v1i64)
10261 return Register();
10262 if ((Subtarget->isNeonAvailable())) {
10263 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
10264 }
10265 return Register();
10266}
10267
10268Register fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10269 if (RetVT.SimpleTy != MVT::v2i64)
10270 return Register();
10271 if ((Subtarget->isNeonAvailable())) {
10272 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10273 }
10274 return Register();
10275}
10276
10277Register fastEmit_ISD_ADD_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10278 if (RetVT.SimpleTy != MVT::nxv16i8)
10279 return Register();
10280 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10281 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
10282 }
10283 return Register();
10284}
10285
10286Register fastEmit_ISD_ADD_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10287 if (RetVT.SimpleTy != MVT::nxv8i16)
10288 return Register();
10289 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10290 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10291 }
10292 return Register();
10293}
10294
10295Register fastEmit_ISD_ADD_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10296 if (RetVT.SimpleTy != MVT::nxv4i32)
10297 return Register();
10298 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10299 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10300 }
10301 return Register();
10302}
10303
10304Register fastEmit_ISD_ADD_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10305 if (RetVT.SimpleTy != MVT::nxv2i64)
10306 return Register();
10307 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10308 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10309 }
10310 return Register();
10311}
10312
10313Register fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10314 switch (VT.SimpleTy) {
10315 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
10316 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op1);
10317 case MVT::v8i8: return fastEmit_ISD_ADD_MVT_v8i8_rr(RetVT, Op0, Op1);
10318 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
10319 case MVT::v4i16: return fastEmit_ISD_ADD_MVT_v4i16_rr(RetVT, Op0, Op1);
10320 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
10321 case MVT::v2i32: return fastEmit_ISD_ADD_MVT_v2i32_rr(RetVT, Op0, Op1);
10322 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
10323 case MVT::v1i64: return fastEmit_ISD_ADD_MVT_v1i64_rr(RetVT, Op0, Op1);
10324 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
10325 case MVT::nxv16i8: return fastEmit_ISD_ADD_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10326 case MVT::nxv8i16: return fastEmit_ISD_ADD_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10327 case MVT::nxv4i32: return fastEmit_ISD_ADD_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10328 case MVT::nxv2i64: return fastEmit_ISD_ADD_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10329 default: return Register();
10330 }
10331}
10332
10333// FastEmit functions for ISD::AND.
10334
10335Register fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10336 if (RetVT.SimpleTy != MVT::i32)
10337 return Register();
10338 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
10339}
10340
10341Register fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10342 if (RetVT.SimpleTy != MVT::i64)
10343 return Register();
10344 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
10345}
10346
10347Register fastEmit_ISD_AND_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10348 if (RetVT.SimpleTy != MVT::v8i8)
10349 return Register();
10350 if ((Subtarget->isNeonAvailable())) {
10351 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10352 }
10353 return Register();
10354}
10355
10356Register fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10357 if (RetVT.SimpleTy != MVT::v16i8)
10358 return Register();
10359 if ((Subtarget->isNeonAvailable())) {
10360 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10361 }
10362 return Register();
10363}
10364
10365Register fastEmit_ISD_AND_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10366 if (RetVT.SimpleTy != MVT::v4i16)
10367 return Register();
10368 if ((Subtarget->isNeonAvailable())) {
10369 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10370 }
10371 return Register();
10372}
10373
10374Register fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10375 if (RetVT.SimpleTy != MVT::v8i16)
10376 return Register();
10377 if ((Subtarget->isNeonAvailable())) {
10378 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10379 }
10380 return Register();
10381}
10382
10383Register fastEmit_ISD_AND_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10384 if (RetVT.SimpleTy != MVT::v2i32)
10385 return Register();
10386 if ((Subtarget->isNeonAvailable())) {
10387 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10388 }
10389 return Register();
10390}
10391
10392Register fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10393 if (RetVT.SimpleTy != MVT::v4i32)
10394 return Register();
10395 if ((Subtarget->isNeonAvailable())) {
10396 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10397 }
10398 return Register();
10399}
10400
10401Register fastEmit_ISD_AND_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
10402 if (RetVT.SimpleTy != MVT::v1i64)
10403 return Register();
10404 if ((Subtarget->isNeonAvailable())) {
10405 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10406 }
10407 return Register();
10408}
10409
10410Register fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10411 if (RetVT.SimpleTy != MVT::v2i64)
10412 return Register();
10413 if ((Subtarget->isNeonAvailable())) {
10414 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10415 }
10416 return Register();
10417}
10418
10419Register fastEmit_ISD_AND_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10420 if (RetVT.SimpleTy != MVT::nxv16i8)
10421 return Register();
10422 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10423 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10424 }
10425 return Register();
10426}
10427
10428Register fastEmit_ISD_AND_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10429 if (RetVT.SimpleTy != MVT::nxv8i16)
10430 return Register();
10431 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10432 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10433 }
10434 return Register();
10435}
10436
10437Register fastEmit_ISD_AND_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10438 if (RetVT.SimpleTy != MVT::nxv4i32)
10439 return Register();
10440 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10441 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10442 }
10443 return Register();
10444}
10445
10446Register fastEmit_ISD_AND_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10447 if (RetVT.SimpleTy != MVT::nxv2i64)
10448 return Register();
10449 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10450 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10451 }
10452 return Register();
10453}
10454
10455Register fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10456 switch (VT.SimpleTy) {
10457 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
10458 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op1);
10459 case MVT::v8i8: return fastEmit_ISD_AND_MVT_v8i8_rr(RetVT, Op0, Op1);
10460 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
10461 case MVT::v4i16: return fastEmit_ISD_AND_MVT_v4i16_rr(RetVT, Op0, Op1);
10462 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
10463 case MVT::v2i32: return fastEmit_ISD_AND_MVT_v2i32_rr(RetVT, Op0, Op1);
10464 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
10465 case MVT::v1i64: return fastEmit_ISD_AND_MVT_v1i64_rr(RetVT, Op0, Op1);
10466 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
10467 case MVT::nxv16i8: return fastEmit_ISD_AND_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10468 case MVT::nxv8i16: return fastEmit_ISD_AND_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10469 case MVT::nxv4i32: return fastEmit_ISD_AND_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10470 case MVT::nxv2i64: return fastEmit_ISD_AND_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10471 default: return Register();
10472 }
10473}
10474
10475// FastEmit functions for ISD::AVGCEILS.
10476
10477Register fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10478 if (RetVT.SimpleTy != MVT::v8i8)
10479 return Register();
10480 if ((Subtarget->isNeonAvailable())) {
10481 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10482 }
10483 return Register();
10484}
10485
10486Register fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10487 if (RetVT.SimpleTy != MVT::v16i8)
10488 return Register();
10489 if ((Subtarget->isNeonAvailable())) {
10490 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10491 }
10492 return Register();
10493}
10494
10495Register fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10496 if (RetVT.SimpleTy != MVT::v4i16)
10497 return Register();
10498 if ((Subtarget->isNeonAvailable())) {
10499 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10500 }
10501 return Register();
10502}
10503
10504Register fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10505 if (RetVT.SimpleTy != MVT::v8i16)
10506 return Register();
10507 if ((Subtarget->isNeonAvailable())) {
10508 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10509 }
10510 return Register();
10511}
10512
10513Register fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10514 if (RetVT.SimpleTy != MVT::v2i32)
10515 return Register();
10516 if ((Subtarget->isNeonAvailable())) {
10517 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10518 }
10519 return Register();
10520}
10521
10522Register fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10523 if (RetVT.SimpleTy != MVT::v4i32)
10524 return Register();
10525 if ((Subtarget->isNeonAvailable())) {
10526 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10527 }
10528 return Register();
10529}
10530
10531Register fastEmit_ISD_AVGCEILS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10532 switch (VT.SimpleTy) {
10533 case MVT::v8i8: return fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(RetVT, Op0, Op1);
10534 case MVT::v16i8: return fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(RetVT, Op0, Op1);
10535 case MVT::v4i16: return fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(RetVT, Op0, Op1);
10536 case MVT::v8i16: return fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(RetVT, Op0, Op1);
10537 case MVT::v2i32: return fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(RetVT, Op0, Op1);
10538 case MVT::v4i32: return fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(RetVT, Op0, Op1);
10539 default: return Register();
10540 }
10541}
10542
10543// FastEmit functions for ISD::AVGCEILU.
10544
10545Register fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10546 if (RetVT.SimpleTy != MVT::v8i8)
10547 return Register();
10548 if ((Subtarget->isNeonAvailable())) {
10549 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10550 }
10551 return Register();
10552}
10553
10554Register fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10555 if (RetVT.SimpleTy != MVT::v16i8)
10556 return Register();
10557 if ((Subtarget->isNeonAvailable())) {
10558 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10559 }
10560 return Register();
10561}
10562
10563Register fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10564 if (RetVT.SimpleTy != MVT::v4i16)
10565 return Register();
10566 if ((Subtarget->isNeonAvailable())) {
10567 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10568 }
10569 return Register();
10570}
10571
10572Register fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10573 if (RetVT.SimpleTy != MVT::v8i16)
10574 return Register();
10575 if ((Subtarget->isNeonAvailable())) {
10576 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10577 }
10578 return Register();
10579}
10580
10581Register fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10582 if (RetVT.SimpleTy != MVT::v2i32)
10583 return Register();
10584 if ((Subtarget->isNeonAvailable())) {
10585 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10586 }
10587 return Register();
10588}
10589
10590Register fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10591 if (RetVT.SimpleTy != MVT::v4i32)
10592 return Register();
10593 if ((Subtarget->isNeonAvailable())) {
10594 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10595 }
10596 return Register();
10597}
10598
10599Register fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10600 switch (VT.SimpleTy) {
10601 case MVT::v8i8: return fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(RetVT, Op0, Op1);
10602 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
10603 case MVT::v4i16: return fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(RetVT, Op0, Op1);
10604 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
10605 case MVT::v2i32: return fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(RetVT, Op0, Op1);
10606 case MVT::v4i32: return fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(RetVT, Op0, Op1);
10607 default: return Register();
10608 }
10609}
10610
10611// FastEmit functions for ISD::AVGFLOORS.
10612
10613Register fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10614 if (RetVT.SimpleTy != MVT::v8i8)
10615 return Register();
10616 if ((Subtarget->isNeonAvailable())) {
10617 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10618 }
10619 return Register();
10620}
10621
10622Register fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10623 if (RetVT.SimpleTy != MVT::v16i8)
10624 return Register();
10625 if ((Subtarget->isNeonAvailable())) {
10626 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10627 }
10628 return Register();
10629}
10630
10631Register fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10632 if (RetVT.SimpleTy != MVT::v4i16)
10633 return Register();
10634 if ((Subtarget->isNeonAvailable())) {
10635 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10636 }
10637 return Register();
10638}
10639
10640Register fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10641 if (RetVT.SimpleTy != MVT::v8i16)
10642 return Register();
10643 if ((Subtarget->isNeonAvailable())) {
10644 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10645 }
10646 return Register();
10647}
10648
10649Register fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10650 if (RetVT.SimpleTy != MVT::v2i32)
10651 return Register();
10652 if ((Subtarget->isNeonAvailable())) {
10653 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10654 }
10655 return Register();
10656}
10657
10658Register fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10659 if (RetVT.SimpleTy != MVT::v4i32)
10660 return Register();
10661 if ((Subtarget->isNeonAvailable())) {
10662 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10663 }
10664 return Register();
10665}
10666
10667Register fastEmit_ISD_AVGFLOORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10668 switch (VT.SimpleTy) {
10669 case MVT::v8i8: return fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(RetVT, Op0, Op1);
10670 case MVT::v16i8: return fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(RetVT, Op0, Op1);
10671 case MVT::v4i16: return fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(RetVT, Op0, Op1);
10672 case MVT::v8i16: return fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(RetVT, Op0, Op1);
10673 case MVT::v2i32: return fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(RetVT, Op0, Op1);
10674 case MVT::v4i32: return fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(RetVT, Op0, Op1);
10675 default: return Register();
10676 }
10677}
10678
10679// FastEmit functions for ISD::AVGFLOORU.
10680
10681Register fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10682 if (RetVT.SimpleTy != MVT::v8i8)
10683 return Register();
10684 if ((Subtarget->isNeonAvailable())) {
10685 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10686 }
10687 return Register();
10688}
10689
10690Register fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10691 if (RetVT.SimpleTy != MVT::v16i8)
10692 return Register();
10693 if ((Subtarget->isNeonAvailable())) {
10694 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10695 }
10696 return Register();
10697}
10698
10699Register fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10700 if (RetVT.SimpleTy != MVT::v4i16)
10701 return Register();
10702 if ((Subtarget->isNeonAvailable())) {
10703 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10704 }
10705 return Register();
10706}
10707
10708Register fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10709 if (RetVT.SimpleTy != MVT::v8i16)
10710 return Register();
10711 if ((Subtarget->isNeonAvailable())) {
10712 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10713 }
10714 return Register();
10715}
10716
10717Register fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10718 if (RetVT.SimpleTy != MVT::v2i32)
10719 return Register();
10720 if ((Subtarget->isNeonAvailable())) {
10721 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10722 }
10723 return Register();
10724}
10725
10726Register fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10727 if (RetVT.SimpleTy != MVT::v4i32)
10728 return Register();
10729 if ((Subtarget->isNeonAvailable())) {
10730 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10731 }
10732 return Register();
10733}
10734
10735Register fastEmit_ISD_AVGFLOORU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10736 switch (VT.SimpleTy) {
10737 case MVT::v8i8: return fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(RetVT, Op0, Op1);
10738 case MVT::v16i8: return fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(RetVT, Op0, Op1);
10739 case MVT::v4i16: return fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(RetVT, Op0, Op1);
10740 case MVT::v8i16: return fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(RetVT, Op0, Op1);
10741 case MVT::v2i32: return fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(RetVT, Op0, Op1);
10742 case MVT::v4i32: return fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(RetVT, Op0, Op1);
10743 default: return Register();
10744 }
10745}
10746
10747// FastEmit functions for ISD::CLMUL.
10748
10749Register fastEmit_ISD_CLMUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10750 if (RetVT.SimpleTy != MVT::v8i8)
10751 return Register();
10752 if ((Subtarget->isNeonAvailable())) {
10753 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10754 }
10755 return Register();
10756}
10757
10758Register fastEmit_ISD_CLMUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10759 if (RetVT.SimpleTy != MVT::v16i8)
10760 return Register();
10761 if ((Subtarget->isNeonAvailable())) {
10762 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10763 }
10764 return Register();
10765}
10766
10767Register fastEmit_ISD_CLMUL_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10768 if (RetVT.SimpleTy != MVT::nxv16i8)
10769 return Register();
10770 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2() || Subtarget->hasSME()))) {
10771 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMUL_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
10772 }
10773 return Register();
10774}
10775
10776Register fastEmit_ISD_CLMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10777 switch (VT.SimpleTy) {
10778 case MVT::v8i8: return fastEmit_ISD_CLMUL_MVT_v8i8_rr(RetVT, Op0, Op1);
10779 case MVT::v16i8: return fastEmit_ISD_CLMUL_MVT_v16i8_rr(RetVT, Op0, Op1);
10780 case MVT::nxv16i8: return fastEmit_ISD_CLMUL_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10781 default: return Register();
10782 }
10783}
10784
10785// FastEmit functions for ISD::CONCAT_VECTORS.
10786
10787Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(MVT RetVT, Register Op0, Register Op1) {
10788 if (RetVT.SimpleTy != MVT::nxv2i1)
10789 return Register();
10790 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10791 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
10792 }
10793 return Register();
10794}
10795
10796Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
10797 if (RetVT.SimpleTy != MVT::nxv4i1)
10798 return Register();
10799 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10800 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
10801 }
10802 return Register();
10803}
10804
10805Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
10806 if (RetVT.SimpleTy != MVT::nxv8i1)
10807 return Register();
10808 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10809 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
10810 }
10811 return Register();
10812}
10813
10814Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
10815 if (RetVT.SimpleTy != MVT::nxv16i1)
10816 return Register();
10817 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10818 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
10819 }
10820 return Register();
10821}
10822
10823Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
10824 if (RetVT.SimpleTy != MVT::nxv4f16)
10825 return Register();
10826 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10827 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10828 }
10829 return Register();
10830}
10831
10832Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10833 if (RetVT.SimpleTy != MVT::nxv8f16)
10834 return Register();
10835 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10836 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10837 }
10838 return Register();
10839}
10840
10841Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10842 if (RetVT.SimpleTy != MVT::nxv4bf16)
10843 return Register();
10844 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10845 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10846 }
10847 return Register();
10848}
10849
10850Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10851 if (RetVT.SimpleTy != MVT::nxv8bf16)
10852 return Register();
10853 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10854 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10855 }
10856 return Register();
10857}
10858
10859Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10860 if (RetVT.SimpleTy != MVT::nxv4f32)
10861 return Register();
10862 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10863 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10864 }
10865 return Register();
10866}
10867
10868Register fastEmit_ISD_CONCAT_VECTORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10869 switch (VT.SimpleTy) {
10870 case MVT::nxv1i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(RetVT, Op0, Op1);
10871 case MVT::nxv2i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(RetVT, Op0, Op1);
10872 case MVT::nxv4i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(RetVT, Op0, Op1);
10873 case MVT::nxv8i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(RetVT, Op0, Op1);
10874 case MVT::nxv2f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(RetVT, Op0, Op1);
10875 case MVT::nxv4f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(RetVT, Op0, Op1);
10876 case MVT::nxv2bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
10877 case MVT::nxv4bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
10878 case MVT::nxv2f32: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(RetVT, Op0, Op1);
10879 default: return Register();
10880 }
10881}
10882
10883// FastEmit functions for ISD::FADD.
10884
10885Register fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10886 if (RetVT.SimpleTy != MVT::f16)
10887 return Register();
10888 if ((Subtarget->hasFullFP16())) {
10889 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
10890 }
10891 return Register();
10892}
10893
10894Register fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10895 if (RetVT.SimpleTy != MVT::f32)
10896 return Register();
10897 if ((Subtarget->hasFPARMv8())) {
10898 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
10899 }
10900 return Register();
10901}
10902
10903Register fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
10904 if (RetVT.SimpleTy != MVT::f64)
10905 return Register();
10906 if ((Subtarget->hasFPARMv8())) {
10907 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10908 }
10909 return Register();
10910}
10911
10912Register fastEmit_ISD_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10913 if (RetVT.SimpleTy != MVT::v4f16)
10914 return Register();
10915 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10916 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10917 }
10918 return Register();
10919}
10920
10921Register fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10922 if (RetVT.SimpleTy != MVT::v8f16)
10923 return Register();
10924 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10925 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10926 }
10927 return Register();
10928}
10929
10930Register fastEmit_ISD_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10931 if (RetVT.SimpleTy != MVT::v2f32)
10932 return Register();
10933 if ((Subtarget->isNeonAvailable())) {
10934 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10935 }
10936 return Register();
10937}
10938
10939Register fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10940 if (RetVT.SimpleTy != MVT::v4f32)
10941 return Register();
10942 if ((Subtarget->isNeonAvailable())) {
10943 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10944 }
10945 return Register();
10946}
10947
10948Register fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10949 if (RetVT.SimpleTy != MVT::v2f64)
10950 return Register();
10951 if ((Subtarget->isNeonAvailable())) {
10952 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10953 }
10954 return Register();
10955}
10956
10957Register fastEmit_ISD_FADD_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10958 if (RetVT.SimpleTy != MVT::nxv8f16)
10959 return Register();
10960 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10961 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10962 }
10963 return Register();
10964}
10965
10966Register fastEmit_ISD_FADD_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10967 if (RetVT.SimpleTy != MVT::nxv8bf16)
10968 return Register();
10969 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
10970 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFADD_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10971 }
10972 return Register();
10973}
10974
10975Register fastEmit_ISD_FADD_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10976 if (RetVT.SimpleTy != MVT::nxv4f32)
10977 return Register();
10978 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10979 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10980 }
10981 return Register();
10982}
10983
10984Register fastEmit_ISD_FADD_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10985 if (RetVT.SimpleTy != MVT::nxv2f64)
10986 return Register();
10987 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10988 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10989 }
10990 return Register();
10991}
10992
10993Register fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10994 switch (VT.SimpleTy) {
10995 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
10996 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
10997 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
10998 case MVT::v4f16: return fastEmit_ISD_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
10999 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
11000 case MVT::v2f32: return fastEmit_ISD_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
11001 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
11002 case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
11003 case MVT::nxv8f16: return fastEmit_ISD_FADD_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11004 case MVT::nxv8bf16: return fastEmit_ISD_FADD_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11005 case MVT::nxv4f32: return fastEmit_ISD_FADD_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11006 case MVT::nxv2f64: return fastEmit_ISD_FADD_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11007 default: return Register();
11008 }
11009}
11010
11011// FastEmit functions for ISD::FDIV.
11012
11013Register fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11014 if (RetVT.SimpleTy != MVT::f16)
11015 return Register();
11016 if ((Subtarget->hasFullFP16())) {
11017 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11018 }
11019 return Register();
11020}
11021
11022Register fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11023 if (RetVT.SimpleTy != MVT::f32)
11024 return Register();
11025 if ((Subtarget->hasFPARMv8())) {
11026 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11027 }
11028 return Register();
11029}
11030
11031Register fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11032 if (RetVT.SimpleTy != MVT::f64)
11033 return Register();
11034 if ((Subtarget->hasFPARMv8())) {
11035 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11036 }
11037 return Register();
11038}
11039
11040Register fastEmit_ISD_FDIV_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11041 if (RetVT.SimpleTy != MVT::v4f16)
11042 return Register();
11043 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11044 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11045 }
11046 return Register();
11047}
11048
11049Register fastEmit_ISD_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11050 if (RetVT.SimpleTy != MVT::v8f16)
11051 return Register();
11052 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11053 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11054 }
11055 return Register();
11056}
11057
11058Register fastEmit_ISD_FDIV_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11059 if (RetVT.SimpleTy != MVT::v2f32)
11060 return Register();
11061 if ((Subtarget->isNeonAvailable())) {
11062 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11063 }
11064 return Register();
11065}
11066
11067Register fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11068 if (RetVT.SimpleTy != MVT::v4f32)
11069 return Register();
11070 if ((Subtarget->isNeonAvailable())) {
11071 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11072 }
11073 return Register();
11074}
11075
11076Register fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11077 if (RetVT.SimpleTy != MVT::v2f64)
11078 return Register();
11079 if ((Subtarget->isNeonAvailable())) {
11080 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11081 }
11082 return Register();
11083}
11084
11085Register fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11086 switch (VT.SimpleTy) {
11087 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
11088 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
11089 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
11090 case MVT::v4f16: return fastEmit_ISD_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1);
11091 case MVT::v8f16: return fastEmit_ISD_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
11092 case MVT::v2f32: return fastEmit_ISD_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1);
11093 case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
11094 case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
11095 default: return Register();
11096 }
11097}
11098
11099// FastEmit functions for ISD::FMAXIMUM.
11100
11101Register fastEmit_ISD_FMAXIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11102 if (RetVT.SimpleTy != MVT::f16)
11103 return Register();
11104 if ((Subtarget->hasFullFP16())) {
11105 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11106 }
11107 return Register();
11108}
11109
11110Register fastEmit_ISD_FMAXIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11111 if (RetVT.SimpleTy != MVT::f32)
11112 return Register();
11113 if ((Subtarget->hasFPARMv8())) {
11114 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11115 }
11116 return Register();
11117}
11118
11119Register fastEmit_ISD_FMAXIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11120 if (RetVT.SimpleTy != MVT::f64)
11121 return Register();
11122 if ((Subtarget->hasFPARMv8())) {
11123 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11124 }
11125 return Register();
11126}
11127
11128Register fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11129 if (RetVT.SimpleTy != MVT::v4f16)
11130 return Register();
11131 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11132 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11133 }
11134 return Register();
11135}
11136
11137Register fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11138 if (RetVT.SimpleTy != MVT::v8f16)
11139 return Register();
11140 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11141 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11142 }
11143 return Register();
11144}
11145
11146Register fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11147 if (RetVT.SimpleTy != MVT::v2f32)
11148 return Register();
11149 if ((Subtarget->isNeonAvailable())) {
11150 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11151 }
11152 return Register();
11153}
11154
11155Register fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11156 if (RetVT.SimpleTy != MVT::v4f32)
11157 return Register();
11158 if ((Subtarget->isNeonAvailable())) {
11159 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11160 }
11161 return Register();
11162}
11163
11164Register fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11165 if (RetVT.SimpleTy != MVT::v1f64)
11166 return Register();
11167 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11168}
11169
11170Register fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11171 if (RetVT.SimpleTy != MVT::v2f64)
11172 return Register();
11173 if ((Subtarget->isNeonAvailable())) {
11174 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11175 }
11176 return Register();
11177}
11178
11179Register fastEmit_ISD_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11180 switch (VT.SimpleTy) {
11181 case MVT::f16: return fastEmit_ISD_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1);
11182 case MVT::f32: return fastEmit_ISD_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1);
11183 case MVT::f64: return fastEmit_ISD_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1);
11184 case MVT::v4f16: return fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11185 case MVT::v8f16: return fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11186 case MVT::v2f32: return fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11187 case MVT::v4f32: return fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11188 case MVT::v1f64: return fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11189 case MVT::v2f64: return fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11190 default: return Register();
11191 }
11192}
11193
11194// FastEmit functions for ISD::FMAXNUM.
11195
11196Register fastEmit_ISD_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11197 if (RetVT.SimpleTy != MVT::f16)
11198 return Register();
11199 if ((Subtarget->hasFullFP16())) {
11200 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11201 }
11202 return Register();
11203}
11204
11205Register fastEmit_ISD_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11206 if (RetVT.SimpleTy != MVT::f32)
11207 return Register();
11208 if ((Subtarget->hasFPARMv8())) {
11209 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11210 }
11211 return Register();
11212}
11213
11214Register fastEmit_ISD_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11215 if (RetVT.SimpleTy != MVT::f64)
11216 return Register();
11217 if ((Subtarget->hasFPARMv8())) {
11218 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11219 }
11220 return Register();
11221}
11222
11223Register fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11224 if (RetVT.SimpleTy != MVT::v4f16)
11225 return Register();
11226 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11227 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11228 }
11229 return Register();
11230}
11231
11232Register fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11233 if (RetVT.SimpleTy != MVT::v8f16)
11234 return Register();
11235 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11236 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11237 }
11238 return Register();
11239}
11240
11241Register fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11242 if (RetVT.SimpleTy != MVT::v2f32)
11243 return Register();
11244 if ((Subtarget->isNeonAvailable())) {
11245 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11246 }
11247 return Register();
11248}
11249
11250Register fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11251 if (RetVT.SimpleTy != MVT::v4f32)
11252 return Register();
11253 if ((Subtarget->isNeonAvailable())) {
11254 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11255 }
11256 return Register();
11257}
11258
11259Register fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11260 if (RetVT.SimpleTy != MVT::v1f64)
11261 return Register();
11262 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11263}
11264
11265Register fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11266 if (RetVT.SimpleTy != MVT::v2f64)
11267 return Register();
11268 if ((Subtarget->isNeonAvailable())) {
11269 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11270 }
11271 return Register();
11272}
11273
11274Register fastEmit_ISD_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11275 switch (VT.SimpleTy) {
11276 case MVT::f16: return fastEmit_ISD_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
11277 case MVT::f32: return fastEmit_ISD_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
11278 case MVT::f64: return fastEmit_ISD_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
11279 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11280 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11281 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11282 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11283 case MVT::v1f64: return fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11284 case MVT::v2f64: return fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11285 default: return Register();
11286 }
11287}
11288
11289// FastEmit functions for ISD::FMAXNUM_IEEE.
11290
11291Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11292 if (RetVT.SimpleTy != MVT::f16)
11293 return Register();
11294 if ((Subtarget->hasFullFP16())) {
11295 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11296 }
11297 return Register();
11298}
11299
11300Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11301 if (RetVT.SimpleTy != MVT::f32)
11302 return Register();
11303 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11304}
11305
11306Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11307 if (RetVT.SimpleTy != MVT::f64)
11308 return Register();
11309 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11310}
11311
11312Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11313 if (RetVT.SimpleTy != MVT::v4f16)
11314 return Register();
11315 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11316 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11317 }
11318 return Register();
11319}
11320
11321Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11322 if (RetVT.SimpleTy != MVT::v8f16)
11323 return Register();
11324 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11325 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11326 }
11327 return Register();
11328}
11329
11330Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11331 if (RetVT.SimpleTy != MVT::v2f32)
11332 return Register();
11333 if ((Subtarget->isNeonAvailable())) {
11334 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11335 }
11336 return Register();
11337}
11338
11339Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11340 if (RetVT.SimpleTy != MVT::v4f32)
11341 return Register();
11342 if ((Subtarget->isNeonAvailable())) {
11343 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11344 }
11345 return Register();
11346}
11347
11348Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11349 if (RetVT.SimpleTy != MVT::v2f64)
11350 return Register();
11351 if ((Subtarget->isNeonAvailable())) {
11352 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11353 }
11354 return Register();
11355}
11356
11357Register fastEmit_ISD_FMAXNUM_IEEE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11358 switch (VT.SimpleTy) {
11359 case MVT::f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f16_rr(RetVT, Op0, Op1);
11360 case MVT::f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f32_rr(RetVT, Op0, Op1);
11361 case MVT::f64: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f64_rr(RetVT, Op0, Op1);
11362 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f16_rr(RetVT, Op0, Op1);
11363 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v8f16_rr(RetVT, Op0, Op1);
11364 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f32_rr(RetVT, Op0, Op1);
11365 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f32_rr(RetVT, Op0, Op1);
11366 case MVT::v2f64: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f64_rr(RetVT, Op0, Op1);
11367 default: return Register();
11368 }
11369}
11370
11371// FastEmit functions for ISD::FMINIMUM.
11372
11373Register fastEmit_ISD_FMINIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11374 if (RetVT.SimpleTy != MVT::f16)
11375 return Register();
11376 if ((Subtarget->hasFullFP16())) {
11377 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11378 }
11379 return Register();
11380}
11381
11382Register fastEmit_ISD_FMINIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11383 if (RetVT.SimpleTy != MVT::f32)
11384 return Register();
11385 if ((Subtarget->hasFPARMv8())) {
11386 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11387 }
11388 return Register();
11389}
11390
11391Register fastEmit_ISD_FMINIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11392 if (RetVT.SimpleTy != MVT::f64)
11393 return Register();
11394 if ((Subtarget->hasFPARMv8())) {
11395 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11396 }
11397 return Register();
11398}
11399
11400Register fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11401 if (RetVT.SimpleTy != MVT::v4f16)
11402 return Register();
11403 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11404 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11405 }
11406 return Register();
11407}
11408
11409Register fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11410 if (RetVT.SimpleTy != MVT::v8f16)
11411 return Register();
11412 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11413 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11414 }
11415 return Register();
11416}
11417
11418Register fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11419 if (RetVT.SimpleTy != MVT::v2f32)
11420 return Register();
11421 if ((Subtarget->isNeonAvailable())) {
11422 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11423 }
11424 return Register();
11425}
11426
11427Register fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11428 if (RetVT.SimpleTy != MVT::v4f32)
11429 return Register();
11430 if ((Subtarget->isNeonAvailable())) {
11431 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11432 }
11433 return Register();
11434}
11435
11436Register fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11437 if (RetVT.SimpleTy != MVT::v1f64)
11438 return Register();
11439 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11440}
11441
11442Register fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11443 if (RetVT.SimpleTy != MVT::v2f64)
11444 return Register();
11445 if ((Subtarget->isNeonAvailable())) {
11446 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11447 }
11448 return Register();
11449}
11450
11451Register fastEmit_ISD_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11452 switch (VT.SimpleTy) {
11453 case MVT::f16: return fastEmit_ISD_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1);
11454 case MVT::f32: return fastEmit_ISD_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1);
11455 case MVT::f64: return fastEmit_ISD_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1);
11456 case MVT::v4f16: return fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11457 case MVT::v8f16: return fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11458 case MVT::v2f32: return fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11459 case MVT::v4f32: return fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11460 case MVT::v1f64: return fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11461 case MVT::v2f64: return fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11462 default: return Register();
11463 }
11464}
11465
11466// FastEmit functions for ISD::FMINNUM.
11467
11468Register fastEmit_ISD_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11469 if (RetVT.SimpleTy != MVT::f16)
11470 return Register();
11471 if ((Subtarget->hasFullFP16())) {
11472 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11473 }
11474 return Register();
11475}
11476
11477Register fastEmit_ISD_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11478 if (RetVT.SimpleTy != MVT::f32)
11479 return Register();
11480 if ((Subtarget->hasFPARMv8())) {
11481 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11482 }
11483 return Register();
11484}
11485
11486Register fastEmit_ISD_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11487 if (RetVT.SimpleTy != MVT::f64)
11488 return Register();
11489 if ((Subtarget->hasFPARMv8())) {
11490 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11491 }
11492 return Register();
11493}
11494
11495Register fastEmit_ISD_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11496 if (RetVT.SimpleTy != MVT::v4f16)
11497 return Register();
11498 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11499 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11500 }
11501 return Register();
11502}
11503
11504Register fastEmit_ISD_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11505 if (RetVT.SimpleTy != MVT::v8f16)
11506 return Register();
11507 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11508 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11509 }
11510 return Register();
11511}
11512
11513Register fastEmit_ISD_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11514 if (RetVT.SimpleTy != MVT::v2f32)
11515 return Register();
11516 if ((Subtarget->isNeonAvailable())) {
11517 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11518 }
11519 return Register();
11520}
11521
11522Register fastEmit_ISD_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11523 if (RetVT.SimpleTy != MVT::v4f32)
11524 return Register();
11525 if ((Subtarget->isNeonAvailable())) {
11526 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11527 }
11528 return Register();
11529}
11530
11531Register fastEmit_ISD_FMINNUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11532 if (RetVT.SimpleTy != MVT::v1f64)
11533 return Register();
11534 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11535}
11536
11537Register fastEmit_ISD_FMINNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11538 if (RetVT.SimpleTy != MVT::v2f64)
11539 return Register();
11540 if ((Subtarget->isNeonAvailable())) {
11541 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11542 }
11543 return Register();
11544}
11545
11546Register fastEmit_ISD_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11547 switch (VT.SimpleTy) {
11548 case MVT::f16: return fastEmit_ISD_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
11549 case MVT::f32: return fastEmit_ISD_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
11550 case MVT::f64: return fastEmit_ISD_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
11551 case MVT::v4f16: return fastEmit_ISD_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11552 case MVT::v8f16: return fastEmit_ISD_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11553 case MVT::v2f32: return fastEmit_ISD_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11554 case MVT::v4f32: return fastEmit_ISD_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11555 case MVT::v1f64: return fastEmit_ISD_FMINNUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11556 case MVT::v2f64: return fastEmit_ISD_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11557 default: return Register();
11558 }
11559}
11560
11561// FastEmit functions for ISD::FMINNUM_IEEE.
11562
11563Register fastEmit_ISD_FMINNUM_IEEE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11564 if (RetVT.SimpleTy != MVT::f16)
11565 return Register();
11566 if ((Subtarget->hasFullFP16())) {
11567 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11568 }
11569 return Register();
11570}
11571
11572Register fastEmit_ISD_FMINNUM_IEEE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11573 if (RetVT.SimpleTy != MVT::f32)
11574 return Register();
11575 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11576}
11577
11578Register fastEmit_ISD_FMINNUM_IEEE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11579 if (RetVT.SimpleTy != MVT::f64)
11580 return Register();
11581 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11582}
11583
11584Register fastEmit_ISD_FMINNUM_IEEE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11585 if (RetVT.SimpleTy != MVT::v4f16)
11586 return Register();
11587 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11588 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11589 }
11590 return Register();
11591}
11592
11593Register fastEmit_ISD_FMINNUM_IEEE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11594 if (RetVT.SimpleTy != MVT::v8f16)
11595 return Register();
11596 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11597 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11598 }
11599 return Register();
11600}
11601
11602Register fastEmit_ISD_FMINNUM_IEEE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11603 if (RetVT.SimpleTy != MVT::v2f32)
11604 return Register();
11605 if ((Subtarget->isNeonAvailable())) {
11606 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11607 }
11608 return Register();
11609}
11610
11611Register fastEmit_ISD_FMINNUM_IEEE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11612 if (RetVT.SimpleTy != MVT::v4f32)
11613 return Register();
11614 if ((Subtarget->isNeonAvailable())) {
11615 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11616 }
11617 return Register();
11618}
11619
11620Register fastEmit_ISD_FMINNUM_IEEE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11621 if (RetVT.SimpleTy != MVT::v2f64)
11622 return Register();
11623 if ((Subtarget->isNeonAvailable())) {
11624 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11625 }
11626 return Register();
11627}
11628
11629Register fastEmit_ISD_FMINNUM_IEEE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11630 switch (VT.SimpleTy) {
11631 case MVT::f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_f16_rr(RetVT, Op0, Op1);
11632 case MVT::f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_f32_rr(RetVT, Op0, Op1);
11633 case MVT::f64: return fastEmit_ISD_FMINNUM_IEEE_MVT_f64_rr(RetVT, Op0, Op1);
11634 case MVT::v4f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_v4f16_rr(RetVT, Op0, Op1);
11635 case MVT::v8f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_v8f16_rr(RetVT, Op0, Op1);
11636 case MVT::v2f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_v2f32_rr(RetVT, Op0, Op1);
11637 case MVT::v4f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_v4f32_rr(RetVT, Op0, Op1);
11638 case MVT::v2f64: return fastEmit_ISD_FMINNUM_IEEE_MVT_v2f64_rr(RetVT, Op0, Op1);
11639 default: return Register();
11640 }
11641}
11642
11643// FastEmit functions for ISD::FMUL.
11644
11645Register fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11646 if (RetVT.SimpleTy != MVT::f16)
11647 return Register();
11648 if ((Subtarget->hasFullFP16())) {
11649 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11650 }
11651 return Register();
11652}
11653
11654Register fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11655 if (RetVT.SimpleTy != MVT::f32)
11656 return Register();
11657 if ((Subtarget->hasFPARMv8())) {
11658 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11659 }
11660 return Register();
11661}
11662
11663Register fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11664 if (RetVT.SimpleTy != MVT::f64)
11665 return Register();
11666 if ((Subtarget->hasFPARMv8())) {
11667 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11668 }
11669 return Register();
11670}
11671
11672Register fastEmit_ISD_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11673 if (RetVT.SimpleTy != MVT::v4f16)
11674 return Register();
11675 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11676 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11677 }
11678 return Register();
11679}
11680
11681Register fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11682 if (RetVT.SimpleTy != MVT::v8f16)
11683 return Register();
11684 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11685 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11686 }
11687 return Register();
11688}
11689
11690Register fastEmit_ISD_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11691 if (RetVT.SimpleTy != MVT::v2f32)
11692 return Register();
11693 if ((Subtarget->isNeonAvailable())) {
11694 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11695 }
11696 return Register();
11697}
11698
11699Register fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11700 if (RetVT.SimpleTy != MVT::v4f32)
11701 return Register();
11702 if ((Subtarget->isNeonAvailable())) {
11703 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11704 }
11705 return Register();
11706}
11707
11708Register fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11709 if (RetVT.SimpleTy != MVT::v2f64)
11710 return Register();
11711 if ((Subtarget->isNeonAvailable())) {
11712 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11713 }
11714 return Register();
11715}
11716
11717Register fastEmit_ISD_FMUL_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11718 if (RetVT.SimpleTy != MVT::nxv8f16)
11719 return Register();
11720 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11721 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11722 }
11723 return Register();
11724}
11725
11726Register fastEmit_ISD_FMUL_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11727 if (RetVT.SimpleTy != MVT::nxv8bf16)
11728 return Register();
11729 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
11730 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFMUL_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11731 }
11732 return Register();
11733}
11734
11735Register fastEmit_ISD_FMUL_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11736 if (RetVT.SimpleTy != MVT::nxv4f32)
11737 return Register();
11738 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11739 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11740 }
11741 return Register();
11742}
11743
11744Register fastEmit_ISD_FMUL_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11745 if (RetVT.SimpleTy != MVT::nxv2f64)
11746 return Register();
11747 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11748 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11749 }
11750 return Register();
11751}
11752
11753Register fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11754 switch (VT.SimpleTy) {
11755 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
11756 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
11757 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
11758 case MVT::v4f16: return fastEmit_ISD_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
11759 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
11760 case MVT::v2f32: return fastEmit_ISD_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
11761 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
11762 case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
11763 case MVT::nxv8f16: return fastEmit_ISD_FMUL_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11764 case MVT::nxv8bf16: return fastEmit_ISD_FMUL_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11765 case MVT::nxv4f32: return fastEmit_ISD_FMUL_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11766 case MVT::nxv2f64: return fastEmit_ISD_FMUL_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11767 default: return Register();
11768 }
11769}
11770
11771// FastEmit functions for ISD::FSUB.
11772
11773Register fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11774 if (RetVT.SimpleTy != MVT::f16)
11775 return Register();
11776 if ((Subtarget->hasFullFP16())) {
11777 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11778 }
11779 return Register();
11780}
11781
11782Register fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11783 if (RetVT.SimpleTy != MVT::f32)
11784 return Register();
11785 if ((Subtarget->hasFPARMv8())) {
11786 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11787 }
11788 return Register();
11789}
11790
11791Register fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11792 if (RetVT.SimpleTy != MVT::f64)
11793 return Register();
11794 if ((Subtarget->hasFPARMv8())) {
11795 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11796 }
11797 return Register();
11798}
11799
11800Register fastEmit_ISD_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11801 if (RetVT.SimpleTy != MVT::v4f16)
11802 return Register();
11803 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11804 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11805 }
11806 return Register();
11807}
11808
11809Register fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11810 if (RetVT.SimpleTy != MVT::v8f16)
11811 return Register();
11812 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11813 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11814 }
11815 return Register();
11816}
11817
11818Register fastEmit_ISD_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11819 if (RetVT.SimpleTy != MVT::v2f32)
11820 return Register();
11821 if ((Subtarget->isNeonAvailable())) {
11822 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11823 }
11824 return Register();
11825}
11826
11827Register fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11828 if (RetVT.SimpleTy != MVT::v4f32)
11829 return Register();
11830 if ((Subtarget->isNeonAvailable())) {
11831 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11832 }
11833 return Register();
11834}
11835
11836Register fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11837 if (RetVT.SimpleTy != MVT::v2f64)
11838 return Register();
11839 if ((Subtarget->isNeonAvailable())) {
11840 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11841 }
11842 return Register();
11843}
11844
11845Register fastEmit_ISD_FSUB_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11846 if (RetVT.SimpleTy != MVT::nxv8f16)
11847 return Register();
11848 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11849 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11850 }
11851 return Register();
11852}
11853
11854Register fastEmit_ISD_FSUB_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11855 if (RetVT.SimpleTy != MVT::nxv8bf16)
11856 return Register();
11857 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
11858 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFSUB_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11859 }
11860 return Register();
11861}
11862
11863Register fastEmit_ISD_FSUB_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11864 if (RetVT.SimpleTy != MVT::nxv4f32)
11865 return Register();
11866 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11867 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11868 }
11869 return Register();
11870}
11871
11872Register fastEmit_ISD_FSUB_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11873 if (RetVT.SimpleTy != MVT::nxv2f64)
11874 return Register();
11875 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11876 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11877 }
11878 return Register();
11879}
11880
11881Register fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11882 switch (VT.SimpleTy) {
11883 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
11884 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
11885 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
11886 case MVT::v4f16: return fastEmit_ISD_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
11887 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
11888 case MVT::v2f32: return fastEmit_ISD_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
11889 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
11890 case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
11891 case MVT::nxv8f16: return fastEmit_ISD_FSUB_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11892 case MVT::nxv8bf16: return fastEmit_ISD_FSUB_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11893 case MVT::nxv4f32: return fastEmit_ISD_FSUB_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11894 case MVT::nxv2f64: return fastEmit_ISD_FSUB_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11895 default: return Register();
11896 }
11897}
11898
11899// FastEmit functions for ISD::GET_ACTIVE_LANE_MASK.
11900
11901Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv2i1_rr(Register Op0, Register Op1) {
11902 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11903 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_D, RC: &AArch64::PPRRegClass, Op0, Op1);
11904 }
11905 return Register();
11906}
11907
11908Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv4i1_rr(Register Op0, Register Op1) {
11909 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11910 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_S, RC: &AArch64::PPRRegClass, Op0, Op1);
11911 }
11912 return Register();
11913}
11914
11915Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv8i1_rr(Register Op0, Register Op1) {
11916 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11917 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_H, RC: &AArch64::PPRRegClass, Op0, Op1);
11918 }
11919 return Register();
11920}
11921
11922Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv16i1_rr(Register Op0, Register Op1) {
11923 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11924 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_B, RC: &AArch64::PPRRegClass, Op0, Op1);
11925 }
11926 return Register();
11927}
11928
11929Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
11930switch (RetVT.SimpleTy) {
11931 case MVT::nxv2i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv2i1_rr(Op0, Op1);
11932 case MVT::nxv4i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv4i1_rr(Op0, Op1);
11933 case MVT::nxv8i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv8i1_rr(Op0, Op1);
11934 case MVT::nxv16i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv16i1_rr(Op0, Op1);
11935 default: return Register();
11936}
11937}
11938
11939Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv2i1_rr(Register Op0, Register Op1) {
11940 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11941 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_D, RC: &AArch64::PPRRegClass, Op0, Op1);
11942 }
11943 return Register();
11944}
11945
11946Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv4i1_rr(Register Op0, Register Op1) {
11947 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11948 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_S, RC: &AArch64::PPRRegClass, Op0, Op1);
11949 }
11950 return Register();
11951}
11952
11953Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv8i1_rr(Register Op0, Register Op1) {
11954 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11955 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_H, RC: &AArch64::PPRRegClass, Op0, Op1);
11956 }
11957 return Register();
11958}
11959
11960Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv16i1_rr(Register Op0, Register Op1) {
11961 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11962 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_B, RC: &AArch64::PPRRegClass, Op0, Op1);
11963 }
11964 return Register();
11965}
11966
11967Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11968switch (RetVT.SimpleTy) {
11969 case MVT::nxv2i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv2i1_rr(Op0, Op1);
11970 case MVT::nxv4i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv4i1_rr(Op0, Op1);
11971 case MVT::nxv8i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv8i1_rr(Op0, Op1);
11972 case MVT::nxv16i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv16i1_rr(Op0, Op1);
11973 default: return Register();
11974}
11975}
11976
11977Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11978 switch (VT.SimpleTy) {
11979 case MVT::i32: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_rr(RetVT, Op0, Op1);
11980 case MVT::i64: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_rr(RetVT, Op0, Op1);
11981 default: return Register();
11982 }
11983}
11984
11985// FastEmit functions for ISD::MUL.
11986
11987Register fastEmit_ISD_MUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
11988 if (RetVT.SimpleTy != MVT::v8i8)
11989 return Register();
11990 if ((Subtarget->isNeonAvailable())) {
11991 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
11992 }
11993 return Register();
11994}
11995
11996Register fastEmit_ISD_MUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
11997 if (RetVT.SimpleTy != MVT::v16i8)
11998 return Register();
11999 if ((Subtarget->isNeonAvailable())) {
12000 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12001 }
12002 return Register();
12003}
12004
12005Register fastEmit_ISD_MUL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12006 if (RetVT.SimpleTy != MVT::v4i16)
12007 return Register();
12008 if ((Subtarget->isNeonAvailable())) {
12009 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12010 }
12011 return Register();
12012}
12013
12014Register fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12015 if (RetVT.SimpleTy != MVT::v8i16)
12016 return Register();
12017 if ((Subtarget->isNeonAvailable())) {
12018 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12019 }
12020 return Register();
12021}
12022
12023Register fastEmit_ISD_MUL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12024 if (RetVT.SimpleTy != MVT::v2i32)
12025 return Register();
12026 if ((Subtarget->isNeonAvailable())) {
12027 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12028 }
12029 return Register();
12030}
12031
12032Register fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12033 if (RetVT.SimpleTy != MVT::v4i32)
12034 return Register();
12035 if ((Subtarget->isNeonAvailable())) {
12036 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12037 }
12038 return Register();
12039}
12040
12041Register fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12042 switch (VT.SimpleTy) {
12043 case MVT::v8i8: return fastEmit_ISD_MUL_MVT_v8i8_rr(RetVT, Op0, Op1);
12044 case MVT::v16i8: return fastEmit_ISD_MUL_MVT_v16i8_rr(RetVT, Op0, Op1);
12045 case MVT::v4i16: return fastEmit_ISD_MUL_MVT_v4i16_rr(RetVT, Op0, Op1);
12046 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
12047 case MVT::v2i32: return fastEmit_ISD_MUL_MVT_v2i32_rr(RetVT, Op0, Op1);
12048 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
12049 default: return Register();
12050 }
12051}
12052
12053// FastEmit functions for ISD::MULHS.
12054
12055Register fastEmit_ISD_MULHS_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12056 if (RetVT.SimpleTy != MVT::i64)
12057 return Register();
12058 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULHrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12059}
12060
12061Register fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12062 switch (VT.SimpleTy) {
12063 case MVT::i64: return fastEmit_ISD_MULHS_MVT_i64_rr(RetVT, Op0, Op1);
12064 default: return Register();
12065 }
12066}
12067
12068// FastEmit functions for ISD::MULHU.
12069
12070Register fastEmit_ISD_MULHU_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12071 if (RetVT.SimpleTy != MVT::i64)
12072 return Register();
12073 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULHrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12074}
12075
12076Register fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12077 switch (VT.SimpleTy) {
12078 case MVT::i64: return fastEmit_ISD_MULHU_MVT_i64_rr(RetVT, Op0, Op1);
12079 default: return Register();
12080 }
12081}
12082
12083// FastEmit functions for ISD::OR.
12084
12085Register fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12086 if (RetVT.SimpleTy != MVT::i32)
12087 return Register();
12088 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12089}
12090
12091Register fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12092 if (RetVT.SimpleTy != MVT::i64)
12093 return Register();
12094 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12095}
12096
12097Register fastEmit_ISD_OR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12098 if (RetVT.SimpleTy != MVT::v8i8)
12099 return Register();
12100 if ((Subtarget->isNeonAvailable())) {
12101 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12102 }
12103 return Register();
12104}
12105
12106Register fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12107 if (RetVT.SimpleTy != MVT::v16i8)
12108 return Register();
12109 if ((Subtarget->isNeonAvailable())) {
12110 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12111 }
12112 return Register();
12113}
12114
12115Register fastEmit_ISD_OR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12116 if (RetVT.SimpleTy != MVT::v4i16)
12117 return Register();
12118 if ((Subtarget->isNeonAvailable())) {
12119 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12120 }
12121 return Register();
12122}
12123
12124Register fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12125 if (RetVT.SimpleTy != MVT::v8i16)
12126 return Register();
12127 if ((Subtarget->isNeonAvailable())) {
12128 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12129 }
12130 return Register();
12131}
12132
12133Register fastEmit_ISD_OR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12134 if (RetVT.SimpleTy != MVT::v2i32)
12135 return Register();
12136 if ((Subtarget->isNeonAvailable())) {
12137 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12138 }
12139 return Register();
12140}
12141
12142Register fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12143 if (RetVT.SimpleTy != MVT::v4i32)
12144 return Register();
12145 if ((Subtarget->isNeonAvailable())) {
12146 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12147 }
12148 return Register();
12149}
12150
12151Register fastEmit_ISD_OR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12152 if (RetVT.SimpleTy != MVT::v1i64)
12153 return Register();
12154 if ((Subtarget->isNeonAvailable())) {
12155 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12156 }
12157 return Register();
12158}
12159
12160Register fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12161 if (RetVT.SimpleTy != MVT::v2i64)
12162 return Register();
12163 if ((Subtarget->isNeonAvailable())) {
12164 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12165 }
12166 return Register();
12167}
12168
12169Register fastEmit_ISD_OR_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12170 if (RetVT.SimpleTy != MVT::nxv16i8)
12171 return Register();
12172 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12173 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12174 }
12175 return Register();
12176}
12177
12178Register fastEmit_ISD_OR_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12179 if (RetVT.SimpleTy != MVT::nxv8i16)
12180 return Register();
12181 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12182 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12183 }
12184 return Register();
12185}
12186
12187Register fastEmit_ISD_OR_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12188 if (RetVT.SimpleTy != MVT::nxv4i32)
12189 return Register();
12190 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12191 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12192 }
12193 return Register();
12194}
12195
12196Register fastEmit_ISD_OR_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12197 if (RetVT.SimpleTy != MVT::nxv2i64)
12198 return Register();
12199 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12200 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12201 }
12202 return Register();
12203}
12204
12205Register fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12206 switch (VT.SimpleTy) {
12207 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
12208 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op1);
12209 case MVT::v8i8: return fastEmit_ISD_OR_MVT_v8i8_rr(RetVT, Op0, Op1);
12210 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
12211 case MVT::v4i16: return fastEmit_ISD_OR_MVT_v4i16_rr(RetVT, Op0, Op1);
12212 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
12213 case MVT::v2i32: return fastEmit_ISD_OR_MVT_v2i32_rr(RetVT, Op0, Op1);
12214 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
12215 case MVT::v1i64: return fastEmit_ISD_OR_MVT_v1i64_rr(RetVT, Op0, Op1);
12216 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
12217 case MVT::nxv16i8: return fastEmit_ISD_OR_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12218 case MVT::nxv8i16: return fastEmit_ISD_OR_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12219 case MVT::nxv4i32: return fastEmit_ISD_OR_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12220 case MVT::nxv2i64: return fastEmit_ISD_OR_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12221 default: return Register();
12222 }
12223}
12224
12225// FastEmit functions for ISD::ROTR.
12226
12227Register fastEmit_ISD_ROTR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12228 if (RetVT.SimpleTy != MVT::i64)
12229 return Register();
12230 return fastEmitInst_rr(MachineInstOpcode: AArch64::RORVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12231}
12232
12233Register fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12234 switch (VT.SimpleTy) {
12235 case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_rr(RetVT, Op0, Op1);
12236 default: return Register();
12237 }
12238}
12239
12240// FastEmit functions for ISD::SADDSAT.
12241
12242Register fastEmit_ISD_SADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12243 if (RetVT.SimpleTy != MVT::v8i8)
12244 return Register();
12245 if ((Subtarget->isNeonAvailable())) {
12246 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12247 }
12248 return Register();
12249}
12250
12251Register fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12252 if (RetVT.SimpleTy != MVT::v16i8)
12253 return Register();
12254 if ((Subtarget->isNeonAvailable())) {
12255 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12256 }
12257 return Register();
12258}
12259
12260Register fastEmit_ISD_SADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12261 if (RetVT.SimpleTy != MVT::v4i16)
12262 return Register();
12263 if ((Subtarget->isNeonAvailable())) {
12264 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12265 }
12266 return Register();
12267}
12268
12269Register fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12270 if (RetVT.SimpleTy != MVT::v8i16)
12271 return Register();
12272 if ((Subtarget->isNeonAvailable())) {
12273 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12274 }
12275 return Register();
12276}
12277
12278Register fastEmit_ISD_SADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12279 if (RetVT.SimpleTy != MVT::v2i32)
12280 return Register();
12281 if ((Subtarget->isNeonAvailable())) {
12282 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12283 }
12284 return Register();
12285}
12286
12287Register fastEmit_ISD_SADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12288 if (RetVT.SimpleTy != MVT::v4i32)
12289 return Register();
12290 if ((Subtarget->isNeonAvailable())) {
12291 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12292 }
12293 return Register();
12294}
12295
12296Register fastEmit_ISD_SADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12297 if (RetVT.SimpleTy != MVT::v1i64)
12298 return Register();
12299 if ((Subtarget->isNeonAvailable())) {
12300 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
12301 }
12302 return Register();
12303}
12304
12305Register fastEmit_ISD_SADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12306 if (RetVT.SimpleTy != MVT::v2i64)
12307 return Register();
12308 if ((Subtarget->isNeonAvailable())) {
12309 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12310 }
12311 return Register();
12312}
12313
12314Register fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12315 if (RetVT.SimpleTy != MVT::nxv16i8)
12316 return Register();
12317 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12318 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
12319 }
12320 return Register();
12321}
12322
12323Register fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12324 if (RetVT.SimpleTy != MVT::nxv8i16)
12325 return Register();
12326 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12327 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
12328 }
12329 return Register();
12330}
12331
12332Register fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12333 if (RetVT.SimpleTy != MVT::nxv4i32)
12334 return Register();
12335 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12336 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
12337 }
12338 return Register();
12339}
12340
12341Register fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12342 if (RetVT.SimpleTy != MVT::nxv2i64)
12343 return Register();
12344 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12345 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
12346 }
12347 return Register();
12348}
12349
12350Register fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12351 switch (VT.SimpleTy) {
12352 case MVT::v8i8: return fastEmit_ISD_SADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
12353 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12354 case MVT::v4i16: return fastEmit_ISD_SADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
12355 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12356 case MVT::v2i32: return fastEmit_ISD_SADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
12357 case MVT::v4i32: return fastEmit_ISD_SADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
12358 case MVT::v1i64: return fastEmit_ISD_SADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
12359 case MVT::v2i64: return fastEmit_ISD_SADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
12360 case MVT::nxv16i8: return fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12361 case MVT::nxv8i16: return fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12362 case MVT::nxv4i32: return fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12363 case MVT::nxv2i64: return fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12364 default: return Register();
12365 }
12366}
12367
12368// FastEmit functions for ISD::SDIV.
12369
12370Register fastEmit_ISD_SDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12371 if (RetVT.SimpleTy != MVT::i32)
12372 return Register();
12373 return fastEmitInst_rr(MachineInstOpcode: AArch64::SDIVWr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12374}
12375
12376Register fastEmit_ISD_SDIV_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12377 if (RetVT.SimpleTy != MVT::i64)
12378 return Register();
12379 return fastEmitInst_rr(MachineInstOpcode: AArch64::SDIVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12380}
12381
12382Register fastEmit_ISD_SDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12383 switch (VT.SimpleTy) {
12384 case MVT::i32: return fastEmit_ISD_SDIV_MVT_i32_rr(RetVT, Op0, Op1);
12385 case MVT::i64: return fastEmit_ISD_SDIV_MVT_i64_rr(RetVT, Op0, Op1);
12386 default: return Register();
12387 }
12388}
12389
12390// FastEmit functions for ISD::SHL.
12391
12392Register fastEmit_ISD_SHL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12393 if (RetVT.SimpleTy != MVT::i64)
12394 return Register();
12395 return fastEmitInst_rr(MachineInstOpcode: AArch64::LSLVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12396}
12397
12398Register fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12399 switch (VT.SimpleTy) {
12400 case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_rr(RetVT, Op0, Op1);
12401 default: return Register();
12402 }
12403}
12404
12405// FastEmit functions for ISD::SMAX.
12406
12407Register fastEmit_ISD_SMAX_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12408 if (RetVT.SimpleTy != MVT::i32)
12409 return Register();
12410 if ((Subtarget->hasCSSC())) {
12411 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12412 }
12413 return Register();
12414}
12415
12416Register fastEmit_ISD_SMAX_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12417 if (RetVT.SimpleTy != MVT::i64)
12418 return Register();
12419 if ((Subtarget->hasCSSC())) {
12420 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12421 }
12422 return Register();
12423}
12424
12425Register fastEmit_ISD_SMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12426 if (RetVT.SimpleTy != MVT::v8i8)
12427 return Register();
12428 if ((Subtarget->isNeonAvailable())) {
12429 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12430 }
12431 return Register();
12432}
12433
12434Register fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12435 if (RetVT.SimpleTy != MVT::v16i8)
12436 return Register();
12437 if ((Subtarget->isNeonAvailable())) {
12438 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12439 }
12440 return Register();
12441}
12442
12443Register fastEmit_ISD_SMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12444 if (RetVT.SimpleTy != MVT::v4i16)
12445 return Register();
12446 if ((Subtarget->isNeonAvailable())) {
12447 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12448 }
12449 return Register();
12450}
12451
12452Register fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12453 if (RetVT.SimpleTy != MVT::v8i16)
12454 return Register();
12455 if ((Subtarget->isNeonAvailable())) {
12456 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12457 }
12458 return Register();
12459}
12460
12461Register fastEmit_ISD_SMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12462 if (RetVT.SimpleTy != MVT::v2i32)
12463 return Register();
12464 if ((Subtarget->isNeonAvailable())) {
12465 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12466 }
12467 return Register();
12468}
12469
12470Register fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12471 if (RetVT.SimpleTy != MVT::v4i32)
12472 return Register();
12473 if ((Subtarget->isNeonAvailable())) {
12474 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12475 }
12476 return Register();
12477}
12478
12479Register fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12480 switch (VT.SimpleTy) {
12481 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_rr(RetVT, Op0, Op1);
12482 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_rr(RetVT, Op0, Op1);
12483 case MVT::v8i8: return fastEmit_ISD_SMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
12484 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
12485 case MVT::v4i16: return fastEmit_ISD_SMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
12486 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
12487 case MVT::v2i32: return fastEmit_ISD_SMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
12488 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
12489 default: return Register();
12490 }
12491}
12492
12493// FastEmit functions for ISD::SMIN.
12494
12495Register fastEmit_ISD_SMIN_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12496 if (RetVT.SimpleTy != MVT::i32)
12497 return Register();
12498 if ((Subtarget->hasCSSC())) {
12499 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12500 }
12501 return Register();
12502}
12503
12504Register fastEmit_ISD_SMIN_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12505 if (RetVT.SimpleTy != MVT::i64)
12506 return Register();
12507 if ((Subtarget->hasCSSC())) {
12508 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12509 }
12510 return Register();
12511}
12512
12513Register fastEmit_ISD_SMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12514 if (RetVT.SimpleTy != MVT::v8i8)
12515 return Register();
12516 if ((Subtarget->isNeonAvailable())) {
12517 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12518 }
12519 return Register();
12520}
12521
12522Register fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12523 if (RetVT.SimpleTy != MVT::v16i8)
12524 return Register();
12525 if ((Subtarget->isNeonAvailable())) {
12526 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12527 }
12528 return Register();
12529}
12530
12531Register fastEmit_ISD_SMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12532 if (RetVT.SimpleTy != MVT::v4i16)
12533 return Register();
12534 if ((Subtarget->isNeonAvailable())) {
12535 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12536 }
12537 return Register();
12538}
12539
12540Register fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12541 if (RetVT.SimpleTy != MVT::v8i16)
12542 return Register();
12543 if ((Subtarget->isNeonAvailable())) {
12544 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12545 }
12546 return Register();
12547}
12548
12549Register fastEmit_ISD_SMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12550 if (RetVT.SimpleTy != MVT::v2i32)
12551 return Register();
12552 if ((Subtarget->isNeonAvailable())) {
12553 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12554 }
12555 return Register();
12556}
12557
12558Register fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12559 if (RetVT.SimpleTy != MVT::v4i32)
12560 return Register();
12561 if ((Subtarget->isNeonAvailable())) {
12562 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12563 }
12564 return Register();
12565}
12566
12567Register fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12568 switch (VT.SimpleTy) {
12569 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_rr(RetVT, Op0, Op1);
12570 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_rr(RetVT, Op0, Op1);
12571 case MVT::v8i8: return fastEmit_ISD_SMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
12572 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
12573 case MVT::v4i16: return fastEmit_ISD_SMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
12574 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
12575 case MVT::v2i32: return fastEmit_ISD_SMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
12576 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
12577 default: return Register();
12578 }
12579}
12580
12581// FastEmit functions for ISD::SRA.
12582
12583Register fastEmit_ISD_SRA_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12584 if (RetVT.SimpleTy != MVT::i64)
12585 return Register();
12586 return fastEmitInst_rr(MachineInstOpcode: AArch64::ASRVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12587}
12588
12589Register fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12590 switch (VT.SimpleTy) {
12591 case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_rr(RetVT, Op0, Op1);
12592 default: return Register();
12593 }
12594}
12595
12596// FastEmit functions for ISD::SRL.
12597
12598Register fastEmit_ISD_SRL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12599 if (RetVT.SimpleTy != MVT::i64)
12600 return Register();
12601 return fastEmitInst_rr(MachineInstOpcode: AArch64::LSRVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12602}
12603
12604Register fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12605 switch (VT.SimpleTy) {
12606 case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_rr(RetVT, Op0, Op1);
12607 default: return Register();
12608 }
12609}
12610
12611// FastEmit functions for ISD::SSUBSAT.
12612
12613Register fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12614 if (RetVT.SimpleTy != MVT::v8i8)
12615 return Register();
12616 if ((Subtarget->isNeonAvailable())) {
12617 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12618 }
12619 return Register();
12620}
12621
12622Register fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12623 if (RetVT.SimpleTy != MVT::v16i8)
12624 return Register();
12625 if ((Subtarget->isNeonAvailable())) {
12626 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12627 }
12628 return Register();
12629}
12630
12631Register fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12632 if (RetVT.SimpleTy != MVT::v4i16)
12633 return Register();
12634 if ((Subtarget->isNeonAvailable())) {
12635 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12636 }
12637 return Register();
12638}
12639
12640Register fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12641 if (RetVT.SimpleTy != MVT::v8i16)
12642 return Register();
12643 if ((Subtarget->isNeonAvailable())) {
12644 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12645 }
12646 return Register();
12647}
12648
12649Register fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12650 if (RetVT.SimpleTy != MVT::v2i32)
12651 return Register();
12652 if ((Subtarget->isNeonAvailable())) {
12653 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12654 }
12655 return Register();
12656}
12657
12658Register fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12659 if (RetVT.SimpleTy != MVT::v4i32)
12660 return Register();
12661 if ((Subtarget->isNeonAvailable())) {
12662 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12663 }
12664 return Register();
12665}
12666
12667Register fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12668 if (RetVT.SimpleTy != MVT::v1i64)
12669 return Register();
12670 if ((Subtarget->isNeonAvailable())) {
12671 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
12672 }
12673 return Register();
12674}
12675
12676Register fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12677 if (RetVT.SimpleTy != MVT::v2i64)
12678 return Register();
12679 if ((Subtarget->isNeonAvailable())) {
12680 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12681 }
12682 return Register();
12683}
12684
12685Register fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12686 if (RetVT.SimpleTy != MVT::nxv16i8)
12687 return Register();
12688 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12689 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
12690 }
12691 return Register();
12692}
12693
12694Register fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12695 if (RetVT.SimpleTy != MVT::nxv8i16)
12696 return Register();
12697 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12698 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
12699 }
12700 return Register();
12701}
12702
12703Register fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12704 if (RetVT.SimpleTy != MVT::nxv4i32)
12705 return Register();
12706 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12707 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
12708 }
12709 return Register();
12710}
12711
12712Register fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12713 if (RetVT.SimpleTy != MVT::nxv2i64)
12714 return Register();
12715 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12716 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
12717 }
12718 return Register();
12719}
12720
12721Register fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12722 switch (VT.SimpleTy) {
12723 case MVT::v8i8: return fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
12724 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12725 case MVT::v4i16: return fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
12726 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12727 case MVT::v2i32: return fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
12728 case MVT::v4i32: return fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
12729 case MVT::v1i64: return fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
12730 case MVT::v2i64: return fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
12731 case MVT::nxv16i8: return fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12732 case MVT::nxv8i16: return fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12733 case MVT::nxv4i32: return fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12734 case MVT::nxv2i64: return fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12735 default: return Register();
12736 }
12737}
12738
12739// FastEmit functions for ISD::STRICT_FADD.
12740
12741Register fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12742 if (RetVT.SimpleTy != MVT::f16)
12743 return Register();
12744 if ((Subtarget->hasFullFP16())) {
12745 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12746 }
12747 return Register();
12748}
12749
12750Register fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12751 if (RetVT.SimpleTy != MVT::f32)
12752 return Register();
12753 if ((Subtarget->hasFPARMv8())) {
12754 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12755 }
12756 return Register();
12757}
12758
12759Register fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12760 if (RetVT.SimpleTy != MVT::f64)
12761 return Register();
12762 if ((Subtarget->hasFPARMv8())) {
12763 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12764 }
12765 return Register();
12766}
12767
12768Register fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12769 if (RetVT.SimpleTy != MVT::v4f16)
12770 return Register();
12771 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12772 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12773 }
12774 return Register();
12775}
12776
12777Register fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12778 if (RetVT.SimpleTy != MVT::v8f16)
12779 return Register();
12780 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12781 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12782 }
12783 return Register();
12784}
12785
12786Register fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12787 if (RetVT.SimpleTy != MVT::v2f32)
12788 return Register();
12789 if ((Subtarget->isNeonAvailable())) {
12790 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12791 }
12792 return Register();
12793}
12794
12795Register fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12796 if (RetVT.SimpleTy != MVT::v4f32)
12797 return Register();
12798 if ((Subtarget->isNeonAvailable())) {
12799 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12800 }
12801 return Register();
12802}
12803
12804Register fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12805 if (RetVT.SimpleTy != MVT::v2f64)
12806 return Register();
12807 if ((Subtarget->isNeonAvailable())) {
12808 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12809 }
12810 return Register();
12811}
12812
12813Register fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12814 switch (VT.SimpleTy) {
12815 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1);
12816 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1);
12817 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1);
12818 case MVT::v4f16: return fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
12819 case MVT::v8f16: return fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
12820 case MVT::v2f32: return fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
12821 case MVT::v4f32: return fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
12822 case MVT::v2f64: return fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
12823 default: return Register();
12824 }
12825}
12826
12827// FastEmit functions for ISD::STRICT_FDIV.
12828
12829Register fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12830 if (RetVT.SimpleTy != MVT::f16)
12831 return Register();
12832 if ((Subtarget->hasFullFP16())) {
12833 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12834 }
12835 return Register();
12836}
12837
12838Register fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12839 if (RetVT.SimpleTy != MVT::f32)
12840 return Register();
12841 if ((Subtarget->hasFPARMv8())) {
12842 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12843 }
12844 return Register();
12845}
12846
12847Register fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12848 if (RetVT.SimpleTy != MVT::f64)
12849 return Register();
12850 if ((Subtarget->hasFPARMv8())) {
12851 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12852 }
12853 return Register();
12854}
12855
12856Register fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12857 if (RetVT.SimpleTy != MVT::v4f16)
12858 return Register();
12859 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12860 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12861 }
12862 return Register();
12863}
12864
12865Register fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12866 if (RetVT.SimpleTy != MVT::v8f16)
12867 return Register();
12868 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12869 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12870 }
12871 return Register();
12872}
12873
12874Register fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12875 if (RetVT.SimpleTy != MVT::v2f32)
12876 return Register();
12877 if ((Subtarget->isNeonAvailable())) {
12878 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12879 }
12880 return Register();
12881}
12882
12883Register fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12884 if (RetVT.SimpleTy != MVT::v4f32)
12885 return Register();
12886 if ((Subtarget->isNeonAvailable())) {
12887 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12888 }
12889 return Register();
12890}
12891
12892Register fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12893 if (RetVT.SimpleTy != MVT::v2f64)
12894 return Register();
12895 if ((Subtarget->isNeonAvailable())) {
12896 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12897 }
12898 return Register();
12899}
12900
12901Register fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12902 switch (VT.SimpleTy) {
12903 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
12904 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
12905 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
12906 case MVT::v4f16: return fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1);
12907 case MVT::v8f16: return fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
12908 case MVT::v2f32: return fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1);
12909 case MVT::v4f32: return fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
12910 case MVT::v2f64: return fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
12911 default: return Register();
12912 }
12913}
12914
12915// FastEmit functions for ISD::STRICT_FMAXIMUM.
12916
12917Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12918 if (RetVT.SimpleTy != MVT::f16)
12919 return Register();
12920 if ((Subtarget->hasFullFP16())) {
12921 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12922 }
12923 return Register();
12924}
12925
12926Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12927 if (RetVT.SimpleTy != MVT::f32)
12928 return Register();
12929 if ((Subtarget->hasFPARMv8())) {
12930 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12931 }
12932 return Register();
12933}
12934
12935Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12936 if (RetVT.SimpleTy != MVT::f64)
12937 return Register();
12938 if ((Subtarget->hasFPARMv8())) {
12939 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12940 }
12941 return Register();
12942}
12943
12944Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12945 if (RetVT.SimpleTy != MVT::v4f16)
12946 return Register();
12947 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12948 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12949 }
12950 return Register();
12951}
12952
12953Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12954 if (RetVT.SimpleTy != MVT::v8f16)
12955 return Register();
12956 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12957 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12958 }
12959 return Register();
12960}
12961
12962Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12963 if (RetVT.SimpleTy != MVT::v2f32)
12964 return Register();
12965 if ((Subtarget->isNeonAvailable())) {
12966 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12967 }
12968 return Register();
12969}
12970
12971Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12972 if (RetVT.SimpleTy != MVT::v4f32)
12973 return Register();
12974 if ((Subtarget->isNeonAvailable())) {
12975 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12976 }
12977 return Register();
12978}
12979
12980Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12981 if (RetVT.SimpleTy != MVT::v2f64)
12982 return Register();
12983 if ((Subtarget->isNeonAvailable())) {
12984 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12985 }
12986 return Register();
12987}
12988
12989Register fastEmit_ISD_STRICT_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12990 switch (VT.SimpleTy) {
12991 case MVT::f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1);
12992 case MVT::f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1);
12993 case MVT::f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1);
12994 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
12995 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
12996 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
12997 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
12998 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
12999 default: return Register();
13000 }
13001}
13002
13003// FastEmit functions for ISD::STRICT_FMAXNUM.
13004
13005Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13006 if (RetVT.SimpleTy != MVT::f16)
13007 return Register();
13008 if ((Subtarget->hasFullFP16())) {
13009 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13010 }
13011 return Register();
13012}
13013
13014Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13015 if (RetVT.SimpleTy != MVT::f32)
13016 return Register();
13017 if ((Subtarget->hasFPARMv8())) {
13018 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13019 }
13020 return Register();
13021}
13022
13023Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13024 if (RetVT.SimpleTy != MVT::f64)
13025 return Register();
13026 if ((Subtarget->hasFPARMv8())) {
13027 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13028 }
13029 return Register();
13030}
13031
13032Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13033 if (RetVT.SimpleTy != MVT::v4f16)
13034 return Register();
13035 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13036 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13037 }
13038 return Register();
13039}
13040
13041Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13042 if (RetVT.SimpleTy != MVT::v8f16)
13043 return Register();
13044 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13045 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13046 }
13047 return Register();
13048}
13049
13050Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13051 if (RetVT.SimpleTy != MVT::v2f32)
13052 return Register();
13053 if ((Subtarget->isNeonAvailable())) {
13054 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13055 }
13056 return Register();
13057}
13058
13059Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13060 if (RetVT.SimpleTy != MVT::v4f32)
13061 return Register();
13062 if ((Subtarget->isNeonAvailable())) {
13063 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13064 }
13065 return Register();
13066}
13067
13068Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13069 if (RetVT.SimpleTy != MVT::v2f64)
13070 return Register();
13071 if ((Subtarget->isNeonAvailable())) {
13072 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13073 }
13074 return Register();
13075}
13076
13077Register fastEmit_ISD_STRICT_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13078 switch (VT.SimpleTy) {
13079 case MVT::f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
13080 case MVT::f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
13081 case MVT::f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
13082 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13083 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13084 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13085 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13086 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13087 default: return Register();
13088 }
13089}
13090
13091// FastEmit functions for ISD::STRICT_FMINIMUM.
13092
13093Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13094 if (RetVT.SimpleTy != MVT::f16)
13095 return Register();
13096 if ((Subtarget->hasFullFP16())) {
13097 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13098 }
13099 return Register();
13100}
13101
13102Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13103 if (RetVT.SimpleTy != MVT::f32)
13104 return Register();
13105 if ((Subtarget->hasFPARMv8())) {
13106 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13107 }
13108 return Register();
13109}
13110
13111Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13112 if (RetVT.SimpleTy != MVT::f64)
13113 return Register();
13114 if ((Subtarget->hasFPARMv8())) {
13115 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13116 }
13117 return Register();
13118}
13119
13120Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13121 if (RetVT.SimpleTy != MVT::v4f16)
13122 return Register();
13123 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13124 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13125 }
13126 return Register();
13127}
13128
13129Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13130 if (RetVT.SimpleTy != MVT::v8f16)
13131 return Register();
13132 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13133 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13134 }
13135 return Register();
13136}
13137
13138Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13139 if (RetVT.SimpleTy != MVT::v2f32)
13140 return Register();
13141 if ((Subtarget->isNeonAvailable())) {
13142 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13143 }
13144 return Register();
13145}
13146
13147Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13148 if (RetVT.SimpleTy != MVT::v4f32)
13149 return Register();
13150 if ((Subtarget->isNeonAvailable())) {
13151 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13152 }
13153 return Register();
13154}
13155
13156Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13157 if (RetVT.SimpleTy != MVT::v2f64)
13158 return Register();
13159 if ((Subtarget->isNeonAvailable())) {
13160 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13161 }
13162 return Register();
13163}
13164
13165Register fastEmit_ISD_STRICT_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13166 switch (VT.SimpleTy) {
13167 case MVT::f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1);
13168 case MVT::f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1);
13169 case MVT::f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1);
13170 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13171 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13172 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13173 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13174 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13175 default: return Register();
13176 }
13177}
13178
13179// FastEmit functions for ISD::STRICT_FMINNUM.
13180
13181Register fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13182 if (RetVT.SimpleTy != MVT::f16)
13183 return Register();
13184 if ((Subtarget->hasFullFP16())) {
13185 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13186 }
13187 return Register();
13188}
13189
13190Register fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13191 if (RetVT.SimpleTy != MVT::f32)
13192 return Register();
13193 if ((Subtarget->hasFPARMv8())) {
13194 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13195 }
13196 return Register();
13197}
13198
13199Register fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13200 if (RetVT.SimpleTy != MVT::f64)
13201 return Register();
13202 if ((Subtarget->hasFPARMv8())) {
13203 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13204 }
13205 return Register();
13206}
13207
13208Register fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13209 if (RetVT.SimpleTy != MVT::v4f16)
13210 return Register();
13211 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13212 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13213 }
13214 return Register();
13215}
13216
13217Register fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13218 if (RetVT.SimpleTy != MVT::v8f16)
13219 return Register();
13220 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13221 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13222 }
13223 return Register();
13224}
13225
13226Register fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13227 if (RetVT.SimpleTy != MVT::v2f32)
13228 return Register();
13229 if ((Subtarget->isNeonAvailable())) {
13230 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13231 }
13232 return Register();
13233}
13234
13235Register fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13236 if (RetVT.SimpleTy != MVT::v4f32)
13237 return Register();
13238 if ((Subtarget->isNeonAvailable())) {
13239 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13240 }
13241 return Register();
13242}
13243
13244Register fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13245 if (RetVT.SimpleTy != MVT::v2f64)
13246 return Register();
13247 if ((Subtarget->isNeonAvailable())) {
13248 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13249 }
13250 return Register();
13251}
13252
13253Register fastEmit_ISD_STRICT_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13254 switch (VT.SimpleTy) {
13255 case MVT::f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
13256 case MVT::f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
13257 case MVT::f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
13258 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13259 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13260 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13261 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13262 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13263 default: return Register();
13264 }
13265}
13266
13267// FastEmit functions for ISD::STRICT_FMUL.
13268
13269Register fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13270 if (RetVT.SimpleTy != MVT::f16)
13271 return Register();
13272 if ((Subtarget->hasFullFP16())) {
13273 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13274 }
13275 return Register();
13276}
13277
13278Register fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13279 if (RetVT.SimpleTy != MVT::f32)
13280 return Register();
13281 if ((Subtarget->hasFPARMv8())) {
13282 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13283 }
13284 return Register();
13285}
13286
13287Register fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13288 if (RetVT.SimpleTy != MVT::f64)
13289 return Register();
13290 if ((Subtarget->hasFPARMv8())) {
13291 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13292 }
13293 return Register();
13294}
13295
13296Register fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13297 if (RetVT.SimpleTy != MVT::v4f16)
13298 return Register();
13299 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13300 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13301 }
13302 return Register();
13303}
13304
13305Register fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13306 if (RetVT.SimpleTy != MVT::v8f16)
13307 return Register();
13308 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13309 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13310 }
13311 return Register();
13312}
13313
13314Register fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13315 if (RetVT.SimpleTy != MVT::v2f32)
13316 return Register();
13317 if ((Subtarget->isNeonAvailable())) {
13318 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13319 }
13320 return Register();
13321}
13322
13323Register fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13324 if (RetVT.SimpleTy != MVT::v4f32)
13325 return Register();
13326 if ((Subtarget->isNeonAvailable())) {
13327 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13328 }
13329 return Register();
13330}
13331
13332Register fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13333 if (RetVT.SimpleTy != MVT::v2f64)
13334 return Register();
13335 if ((Subtarget->isNeonAvailable())) {
13336 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13337 }
13338 return Register();
13339}
13340
13341Register fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13342 switch (VT.SimpleTy) {
13343 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
13344 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
13345 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
13346 case MVT::v4f16: return fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
13347 case MVT::v8f16: return fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
13348 case MVT::v2f32: return fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
13349 case MVT::v4f32: return fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
13350 case MVT::v2f64: return fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
13351 default: return Register();
13352 }
13353}
13354
13355// FastEmit functions for ISD::STRICT_FSUB.
13356
13357Register fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13358 if (RetVT.SimpleTy != MVT::f16)
13359 return Register();
13360 if ((Subtarget->hasFullFP16())) {
13361 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13362 }
13363 return Register();
13364}
13365
13366Register fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13367 if (RetVT.SimpleTy != MVT::f32)
13368 return Register();
13369 if ((Subtarget->hasFPARMv8())) {
13370 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13371 }
13372 return Register();
13373}
13374
13375Register fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13376 if (RetVT.SimpleTy != MVT::f64)
13377 return Register();
13378 if ((Subtarget->hasFPARMv8())) {
13379 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13380 }
13381 return Register();
13382}
13383
13384Register fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13385 if (RetVT.SimpleTy != MVT::v4f16)
13386 return Register();
13387 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13388 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13389 }
13390 return Register();
13391}
13392
13393Register fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13394 if (RetVT.SimpleTy != MVT::v8f16)
13395 return Register();
13396 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13397 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13398 }
13399 return Register();
13400}
13401
13402Register fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13403 if (RetVT.SimpleTy != MVT::v2f32)
13404 return Register();
13405 if ((Subtarget->isNeonAvailable())) {
13406 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13407 }
13408 return Register();
13409}
13410
13411Register fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13412 if (RetVT.SimpleTy != MVT::v4f32)
13413 return Register();
13414 if ((Subtarget->isNeonAvailable())) {
13415 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13416 }
13417 return Register();
13418}
13419
13420Register fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13421 if (RetVT.SimpleTy != MVT::v2f64)
13422 return Register();
13423 if ((Subtarget->isNeonAvailable())) {
13424 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13425 }
13426 return Register();
13427}
13428
13429Register fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13430 switch (VT.SimpleTy) {
13431 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
13432 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
13433 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
13434 case MVT::v4f16: return fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
13435 case MVT::v8f16: return fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
13436 case MVT::v2f32: return fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
13437 case MVT::v4f32: return fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
13438 case MVT::v2f64: return fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
13439 default: return Register();
13440 }
13441}
13442
13443// FastEmit functions for ISD::SUB.
13444
13445Register fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13446 if (RetVT.SimpleTy != MVT::i32)
13447 return Register();
13448 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBSWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13449}
13450
13451Register fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13452 if (RetVT.SimpleTy != MVT::i64)
13453 return Register();
13454 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBSXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13455}
13456
13457Register fastEmit_ISD_SUB_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13458 if (RetVT.SimpleTy != MVT::v8i8)
13459 return Register();
13460 if ((Subtarget->isNeonAvailable())) {
13461 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13462 }
13463 return Register();
13464}
13465
13466Register fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13467 if (RetVT.SimpleTy != MVT::v16i8)
13468 return Register();
13469 if ((Subtarget->isNeonAvailable())) {
13470 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13471 }
13472 return Register();
13473}
13474
13475Register fastEmit_ISD_SUB_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13476 if (RetVT.SimpleTy != MVT::v4i16)
13477 return Register();
13478 if ((Subtarget->isNeonAvailable())) {
13479 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13480 }
13481 return Register();
13482}
13483
13484Register fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13485 if (RetVT.SimpleTy != MVT::v8i16)
13486 return Register();
13487 if ((Subtarget->isNeonAvailable())) {
13488 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13489 }
13490 return Register();
13491}
13492
13493Register fastEmit_ISD_SUB_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13494 if (RetVT.SimpleTy != MVT::v2i32)
13495 return Register();
13496 if ((Subtarget->isNeonAvailable())) {
13497 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13498 }
13499 return Register();
13500}
13501
13502Register fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13503 if (RetVT.SimpleTy != MVT::v4i32)
13504 return Register();
13505 if ((Subtarget->isNeonAvailable())) {
13506 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13507 }
13508 return Register();
13509}
13510
13511Register fastEmit_ISD_SUB_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13512 if (RetVT.SimpleTy != MVT::v1i64)
13513 return Register();
13514 if ((Subtarget->isNeonAvailable())) {
13515 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13516 }
13517 return Register();
13518}
13519
13520Register fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13521 if (RetVT.SimpleTy != MVT::v2i64)
13522 return Register();
13523 if ((Subtarget->isNeonAvailable())) {
13524 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13525 }
13526 return Register();
13527}
13528
13529Register fastEmit_ISD_SUB_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13530 if (RetVT.SimpleTy != MVT::nxv16i8)
13531 return Register();
13532 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13533 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13534 }
13535 return Register();
13536}
13537
13538Register fastEmit_ISD_SUB_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13539 if (RetVT.SimpleTy != MVT::nxv8i16)
13540 return Register();
13541 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13542 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13543 }
13544 return Register();
13545}
13546
13547Register fastEmit_ISD_SUB_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13548 if (RetVT.SimpleTy != MVT::nxv4i32)
13549 return Register();
13550 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13551 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13552 }
13553 return Register();
13554}
13555
13556Register fastEmit_ISD_SUB_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13557 if (RetVT.SimpleTy != MVT::nxv2i64)
13558 return Register();
13559 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13560 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13561 }
13562 return Register();
13563}
13564
13565Register fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13566 switch (VT.SimpleTy) {
13567 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
13568 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op1);
13569 case MVT::v8i8: return fastEmit_ISD_SUB_MVT_v8i8_rr(RetVT, Op0, Op1);
13570 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
13571 case MVT::v4i16: return fastEmit_ISD_SUB_MVT_v4i16_rr(RetVT, Op0, Op1);
13572 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
13573 case MVT::v2i32: return fastEmit_ISD_SUB_MVT_v2i32_rr(RetVT, Op0, Op1);
13574 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
13575 case MVT::v1i64: return fastEmit_ISD_SUB_MVT_v1i64_rr(RetVT, Op0, Op1);
13576 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
13577 case MVT::nxv16i8: return fastEmit_ISD_SUB_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13578 case MVT::nxv8i16: return fastEmit_ISD_SUB_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13579 case MVT::nxv4i32: return fastEmit_ISD_SUB_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13580 case MVT::nxv2i64: return fastEmit_ISD_SUB_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13581 default: return Register();
13582 }
13583}
13584
13585// FastEmit functions for ISD::UADDSAT.
13586
13587Register fastEmit_ISD_UADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13588 if (RetVT.SimpleTy != MVT::v8i8)
13589 return Register();
13590 if ((Subtarget->isNeonAvailable())) {
13591 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13592 }
13593 return Register();
13594}
13595
13596Register fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13597 if (RetVT.SimpleTy != MVT::v16i8)
13598 return Register();
13599 if ((Subtarget->isNeonAvailable())) {
13600 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13601 }
13602 return Register();
13603}
13604
13605Register fastEmit_ISD_UADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13606 if (RetVT.SimpleTy != MVT::v4i16)
13607 return Register();
13608 if ((Subtarget->isNeonAvailable())) {
13609 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13610 }
13611 return Register();
13612}
13613
13614Register fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13615 if (RetVT.SimpleTy != MVT::v8i16)
13616 return Register();
13617 if ((Subtarget->isNeonAvailable())) {
13618 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13619 }
13620 return Register();
13621}
13622
13623Register fastEmit_ISD_UADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13624 if (RetVT.SimpleTy != MVT::v2i32)
13625 return Register();
13626 if ((Subtarget->isNeonAvailable())) {
13627 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13628 }
13629 return Register();
13630}
13631
13632Register fastEmit_ISD_UADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13633 if (RetVT.SimpleTy != MVT::v4i32)
13634 return Register();
13635 if ((Subtarget->isNeonAvailable())) {
13636 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13637 }
13638 return Register();
13639}
13640
13641Register fastEmit_ISD_UADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13642 if (RetVT.SimpleTy != MVT::v1i64)
13643 return Register();
13644 if ((Subtarget->isNeonAvailable())) {
13645 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13646 }
13647 return Register();
13648}
13649
13650Register fastEmit_ISD_UADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13651 if (RetVT.SimpleTy != MVT::v2i64)
13652 return Register();
13653 if ((Subtarget->isNeonAvailable())) {
13654 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13655 }
13656 return Register();
13657}
13658
13659Register fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13660 if (RetVT.SimpleTy != MVT::nxv16i8)
13661 return Register();
13662 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13663 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13664 }
13665 return Register();
13666}
13667
13668Register fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13669 if (RetVT.SimpleTy != MVT::nxv8i16)
13670 return Register();
13671 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13672 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13673 }
13674 return Register();
13675}
13676
13677Register fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13678 if (RetVT.SimpleTy != MVT::nxv4i32)
13679 return Register();
13680 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13681 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13682 }
13683 return Register();
13684}
13685
13686Register fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13687 if (RetVT.SimpleTy != MVT::nxv2i64)
13688 return Register();
13689 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13690 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13691 }
13692 return Register();
13693}
13694
13695Register fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13696 switch (VT.SimpleTy) {
13697 case MVT::v8i8: return fastEmit_ISD_UADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
13698 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13699 case MVT::v4i16: return fastEmit_ISD_UADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
13700 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13701 case MVT::v2i32: return fastEmit_ISD_UADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
13702 case MVT::v4i32: return fastEmit_ISD_UADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
13703 case MVT::v1i64: return fastEmit_ISD_UADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
13704 case MVT::v2i64: return fastEmit_ISD_UADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
13705 case MVT::nxv16i8: return fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13706 case MVT::nxv8i16: return fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13707 case MVT::nxv4i32: return fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13708 case MVT::nxv2i64: return fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13709 default: return Register();
13710 }
13711}
13712
13713// FastEmit functions for ISD::UDIV.
13714
13715Register fastEmit_ISD_UDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13716 if (RetVT.SimpleTy != MVT::i32)
13717 return Register();
13718 return fastEmitInst_rr(MachineInstOpcode: AArch64::UDIVWr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13719}
13720
13721Register fastEmit_ISD_UDIV_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13722 if (RetVT.SimpleTy != MVT::i64)
13723 return Register();
13724 return fastEmitInst_rr(MachineInstOpcode: AArch64::UDIVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13725}
13726
13727Register fastEmit_ISD_UDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13728 switch (VT.SimpleTy) {
13729 case MVT::i32: return fastEmit_ISD_UDIV_MVT_i32_rr(RetVT, Op0, Op1);
13730 case MVT::i64: return fastEmit_ISD_UDIV_MVT_i64_rr(RetVT, Op0, Op1);
13731 default: return Register();
13732 }
13733}
13734
13735// FastEmit functions for ISD::UMAX.
13736
13737Register fastEmit_ISD_UMAX_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13738 if (RetVT.SimpleTy != MVT::i32)
13739 return Register();
13740 if ((Subtarget->hasCSSC())) {
13741 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13742 }
13743 return Register();
13744}
13745
13746Register fastEmit_ISD_UMAX_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13747 if (RetVT.SimpleTy != MVT::i64)
13748 return Register();
13749 if ((Subtarget->hasCSSC())) {
13750 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13751 }
13752 return Register();
13753}
13754
13755Register fastEmit_ISD_UMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13756 if (RetVT.SimpleTy != MVT::v8i8)
13757 return Register();
13758 if ((Subtarget->isNeonAvailable())) {
13759 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13760 }
13761 return Register();
13762}
13763
13764Register fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13765 if (RetVT.SimpleTy != MVT::v16i8)
13766 return Register();
13767 if ((Subtarget->isNeonAvailable())) {
13768 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13769 }
13770 return Register();
13771}
13772
13773Register fastEmit_ISD_UMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13774 if (RetVT.SimpleTy != MVT::v4i16)
13775 return Register();
13776 if ((Subtarget->isNeonAvailable())) {
13777 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13778 }
13779 return Register();
13780}
13781
13782Register fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13783 if (RetVT.SimpleTy != MVT::v8i16)
13784 return Register();
13785 if ((Subtarget->isNeonAvailable())) {
13786 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13787 }
13788 return Register();
13789}
13790
13791Register fastEmit_ISD_UMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13792 if (RetVT.SimpleTy != MVT::v2i32)
13793 return Register();
13794 if ((Subtarget->isNeonAvailable())) {
13795 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13796 }
13797 return Register();
13798}
13799
13800Register fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13801 if (RetVT.SimpleTy != MVT::v4i32)
13802 return Register();
13803 if ((Subtarget->isNeonAvailable())) {
13804 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13805 }
13806 return Register();
13807}
13808
13809Register fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13810 switch (VT.SimpleTy) {
13811 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_rr(RetVT, Op0, Op1);
13812 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_rr(RetVT, Op0, Op1);
13813 case MVT::v8i8: return fastEmit_ISD_UMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
13814 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
13815 case MVT::v4i16: return fastEmit_ISD_UMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
13816 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
13817 case MVT::v2i32: return fastEmit_ISD_UMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
13818 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
13819 default: return Register();
13820 }
13821}
13822
13823// FastEmit functions for ISD::UMIN.
13824
13825Register fastEmit_ISD_UMIN_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13826 if (RetVT.SimpleTy != MVT::i32)
13827 return Register();
13828 if ((Subtarget->hasCSSC())) {
13829 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13830 }
13831 return Register();
13832}
13833
13834Register fastEmit_ISD_UMIN_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13835 if (RetVT.SimpleTy != MVT::i64)
13836 return Register();
13837 if ((Subtarget->hasCSSC())) {
13838 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13839 }
13840 return Register();
13841}
13842
13843Register fastEmit_ISD_UMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13844 if (RetVT.SimpleTy != MVT::v8i8)
13845 return Register();
13846 if ((Subtarget->isNeonAvailable())) {
13847 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13848 }
13849 return Register();
13850}
13851
13852Register fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13853 if (RetVT.SimpleTy != MVT::v16i8)
13854 return Register();
13855 if ((Subtarget->isNeonAvailable())) {
13856 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13857 }
13858 return Register();
13859}
13860
13861Register fastEmit_ISD_UMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13862 if (RetVT.SimpleTy != MVT::v4i16)
13863 return Register();
13864 if ((Subtarget->isNeonAvailable())) {
13865 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13866 }
13867 return Register();
13868}
13869
13870Register fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13871 if (RetVT.SimpleTy != MVT::v8i16)
13872 return Register();
13873 if ((Subtarget->isNeonAvailable())) {
13874 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13875 }
13876 return Register();
13877}
13878
13879Register fastEmit_ISD_UMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13880 if (RetVT.SimpleTy != MVT::v2i32)
13881 return Register();
13882 if ((Subtarget->isNeonAvailable())) {
13883 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13884 }
13885 return Register();
13886}
13887
13888Register fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13889 if (RetVT.SimpleTy != MVT::v4i32)
13890 return Register();
13891 if ((Subtarget->isNeonAvailable())) {
13892 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13893 }
13894 return Register();
13895}
13896
13897Register fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13898 switch (VT.SimpleTy) {
13899 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_rr(RetVT, Op0, Op1);
13900 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_rr(RetVT, Op0, Op1);
13901 case MVT::v8i8: return fastEmit_ISD_UMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
13902 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
13903 case MVT::v4i16: return fastEmit_ISD_UMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
13904 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
13905 case MVT::v2i32: return fastEmit_ISD_UMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
13906 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
13907 default: return Register();
13908 }
13909}
13910
13911// FastEmit functions for ISD::USUBSAT.
13912
13913Register fastEmit_ISD_USUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13914 if (RetVT.SimpleTy != MVT::v8i8)
13915 return Register();
13916 if ((Subtarget->isNeonAvailable())) {
13917 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13918 }
13919 return Register();
13920}
13921
13922Register fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13923 if (RetVT.SimpleTy != MVT::v16i8)
13924 return Register();
13925 if ((Subtarget->isNeonAvailable())) {
13926 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13927 }
13928 return Register();
13929}
13930
13931Register fastEmit_ISD_USUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13932 if (RetVT.SimpleTy != MVT::v4i16)
13933 return Register();
13934 if ((Subtarget->isNeonAvailable())) {
13935 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13936 }
13937 return Register();
13938}
13939
13940Register fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13941 if (RetVT.SimpleTy != MVT::v8i16)
13942 return Register();
13943 if ((Subtarget->isNeonAvailable())) {
13944 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13945 }
13946 return Register();
13947}
13948
13949Register fastEmit_ISD_USUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13950 if (RetVT.SimpleTy != MVT::v2i32)
13951 return Register();
13952 if ((Subtarget->isNeonAvailable())) {
13953 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13954 }
13955 return Register();
13956}
13957
13958Register fastEmit_ISD_USUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13959 if (RetVT.SimpleTy != MVT::v4i32)
13960 return Register();
13961 if ((Subtarget->isNeonAvailable())) {
13962 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13963 }
13964 return Register();
13965}
13966
13967Register fastEmit_ISD_USUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13968 if (RetVT.SimpleTy != MVT::v1i64)
13969 return Register();
13970 if ((Subtarget->isNeonAvailable())) {
13971 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13972 }
13973 return Register();
13974}
13975
13976Register fastEmit_ISD_USUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13977 if (RetVT.SimpleTy != MVT::v2i64)
13978 return Register();
13979 if ((Subtarget->isNeonAvailable())) {
13980 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13981 }
13982 return Register();
13983}
13984
13985Register fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13986 if (RetVT.SimpleTy != MVT::nxv16i8)
13987 return Register();
13988 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13989 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13990 }
13991 return Register();
13992}
13993
13994Register fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13995 if (RetVT.SimpleTy != MVT::nxv8i16)
13996 return Register();
13997 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13998 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13999 }
14000 return Register();
14001}
14002
14003Register fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14004 if (RetVT.SimpleTy != MVT::nxv4i32)
14005 return Register();
14006 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14007 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
14008 }
14009 return Register();
14010}
14011
14012Register fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14013 if (RetVT.SimpleTy != MVT::nxv2i64)
14014 return Register();
14015 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14016 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
14017 }
14018 return Register();
14019}
14020
14021Register fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14022 switch (VT.SimpleTy) {
14023 case MVT::v8i8: return fastEmit_ISD_USUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
14024 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
14025 case MVT::v4i16: return fastEmit_ISD_USUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
14026 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
14027 case MVT::v2i32: return fastEmit_ISD_USUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
14028 case MVT::v4i32: return fastEmit_ISD_USUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
14029 case MVT::v1i64: return fastEmit_ISD_USUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
14030 case MVT::v2i64: return fastEmit_ISD_USUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
14031 case MVT::nxv16i8: return fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
14032 case MVT::nxv8i16: return fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
14033 case MVT::nxv4i32: return fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
14034 case MVT::nxv2i64: return fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
14035 default: return Register();
14036 }
14037}
14038
14039// FastEmit functions for ISD::XOR.
14040
14041Register fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
14042 if (RetVT.SimpleTy != MVT::i32)
14043 return Register();
14044 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
14045}
14046
14047Register fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
14048 if (RetVT.SimpleTy != MVT::i64)
14049 return Register();
14050 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
14051}
14052
14053Register fastEmit_ISD_XOR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
14054 if (RetVT.SimpleTy != MVT::v8i8)
14055 return Register();
14056 if ((Subtarget->isNeonAvailable())) {
14057 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14058 }
14059 return Register();
14060}
14061
14062Register fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14063 if (RetVT.SimpleTy != MVT::v16i8)
14064 return Register();
14065 if ((Subtarget->isNeonAvailable())) {
14066 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14067 }
14068 return Register();
14069}
14070
14071Register fastEmit_ISD_XOR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
14072 if (RetVT.SimpleTy != MVT::v4i16)
14073 return Register();
14074 if ((Subtarget->isNeonAvailable())) {
14075 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14076 }
14077 return Register();
14078}
14079
14080Register fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14081 if (RetVT.SimpleTy != MVT::v8i16)
14082 return Register();
14083 if ((Subtarget->isNeonAvailable())) {
14084 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14085 }
14086 return Register();
14087}
14088
14089Register fastEmit_ISD_XOR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
14090 if (RetVT.SimpleTy != MVT::v2i32)
14091 return Register();
14092 if ((Subtarget->isNeonAvailable())) {
14093 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14094 }
14095 return Register();
14096}
14097
14098Register fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14099 if (RetVT.SimpleTy != MVT::v4i32)
14100 return Register();
14101 if ((Subtarget->isNeonAvailable())) {
14102 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14103 }
14104 return Register();
14105}
14106
14107Register fastEmit_ISD_XOR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
14108 if (RetVT.SimpleTy != MVT::v1i64)
14109 return Register();
14110 if ((Subtarget->isNeonAvailable())) {
14111 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14112 }
14113 return Register();
14114}
14115
14116Register fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14117 if (RetVT.SimpleTy != MVT::v2i64)
14118 return Register();
14119 if ((Subtarget->isNeonAvailable())) {
14120 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14121 }
14122 return Register();
14123}
14124
14125Register fastEmit_ISD_XOR_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14126 if (RetVT.SimpleTy != MVT::nxv16i8)
14127 return Register();
14128 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14129 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14130 }
14131 return Register();
14132}
14133
14134Register fastEmit_ISD_XOR_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14135 if (RetVT.SimpleTy != MVT::nxv8i16)
14136 return Register();
14137 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14138 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14139 }
14140 return Register();
14141}
14142
14143Register fastEmit_ISD_XOR_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14144 if (RetVT.SimpleTy != MVT::nxv4i32)
14145 return Register();
14146 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14147 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14148 }
14149 return Register();
14150}
14151
14152Register fastEmit_ISD_XOR_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14153 if (RetVT.SimpleTy != MVT::nxv2i64)
14154 return Register();
14155 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14156 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14157 }
14158 return Register();
14159}
14160
14161Register fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14162 switch (VT.SimpleTy) {
14163 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
14164 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op1);
14165 case MVT::v8i8: return fastEmit_ISD_XOR_MVT_v8i8_rr(RetVT, Op0, Op1);
14166 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
14167 case MVT::v4i16: return fastEmit_ISD_XOR_MVT_v4i16_rr(RetVT, Op0, Op1);
14168 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
14169 case MVT::v2i32: return fastEmit_ISD_XOR_MVT_v2i32_rr(RetVT, Op0, Op1);
14170 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
14171 case MVT::v1i64: return fastEmit_ISD_XOR_MVT_v1i64_rr(RetVT, Op0, Op1);
14172 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
14173 case MVT::nxv16i8: return fastEmit_ISD_XOR_MVT_nxv16i8_rr(RetVT, Op0, Op1);
14174 case MVT::nxv8i16: return fastEmit_ISD_XOR_MVT_nxv8i16_rr(RetVT, Op0, Op1);
14175 case MVT::nxv4i32: return fastEmit_ISD_XOR_MVT_nxv4i32_rr(RetVT, Op0, Op1);
14176 case MVT::nxv2i64: return fastEmit_ISD_XOR_MVT_nxv2i64_rr(RetVT, Op0, Op1);
14177 default: return Register();
14178 }
14179}
14180
14181// Top-level FastEmit function.
14182
14183Register fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, Register Op1) override {
14184 switch (Opcode) {
14185 case AArch64ISD::ADDP: return fastEmit_AArch64ISD_ADDP_rr(VT, RetVT, Op0, Op1);
14186 case AArch64ISD::BIC: return fastEmit_AArch64ISD_BIC_rr(VT, RetVT, Op0, Op1);
14187 case AArch64ISD::FCMEQ: return fastEmit_AArch64ISD_FCMEQ_rr(VT, RetVT, Op0, Op1);
14188 case AArch64ISD::FCMGE: return fastEmit_AArch64ISD_FCMGE_rr(VT, RetVT, Op0, Op1);
14189 case AArch64ISD::FCMGT: return fastEmit_AArch64ISD_FCMGT_rr(VT, RetVT, Op0, Op1);
14190 case AArch64ISD::FCMP: return fastEmit_AArch64ISD_FCMP_rr(VT, RetVT, Op0, Op1);
14191 case AArch64ISD::FRECPS: return fastEmit_AArch64ISD_FRECPS_rr(VT, RetVT, Op0, Op1);
14192 case AArch64ISD::FRSQRTS: return fastEmit_AArch64ISD_FRSQRTS_rr(VT, RetVT, Op0, Op1);
14193 case AArch64ISD::INIT_TPIDR2OBJ: return fastEmit_AArch64ISD_INIT_TPIDR2OBJ_rr(VT, RetVT, Op0, Op1);
14194 case AArch64ISD::PMULL: return fastEmit_AArch64ISD_PMULL_rr(VT, RetVT, Op0, Op1);
14195 case AArch64ISD::PTEST: return fastEmit_AArch64ISD_PTEST_rr(VT, RetVT, Op0, Op1);
14196 case AArch64ISD::PTEST_ANY: return fastEmit_AArch64ISD_PTEST_ANY_rr(VT, RetVT, Op0, Op1);
14197 case AArch64ISD::PTEST_FIRST: return fastEmit_AArch64ISD_PTEST_FIRST_rr(VT, RetVT, Op0, Op1);
14198 case AArch64ISD::SMULL: return fastEmit_AArch64ISD_SMULL_rr(VT, RetVT, Op0, Op1);
14199 case AArch64ISD::SQADD: return fastEmit_AArch64ISD_SQADD_rr(VT, RetVT, Op0, Op1);
14200 case AArch64ISD::SQDMULH: return fastEmit_AArch64ISD_SQDMULH_rr(VT, RetVT, Op0, Op1);
14201 case AArch64ISD::SQDMULL: return fastEmit_AArch64ISD_SQDMULL_rr(VT, RetVT, Op0, Op1);
14202 case AArch64ISD::SQRDMULH: return fastEmit_AArch64ISD_SQRDMULH_rr(VT, RetVT, Op0, Op1);
14203 case AArch64ISD::SQRSHL: return fastEmit_AArch64ISD_SQRSHL_rr(VT, RetVT, Op0, Op1);
14204 case AArch64ISD::SQSHL: return fastEmit_AArch64ISD_SQSHL_rr(VT, RetVT, Op0, Op1);
14205 case AArch64ISD::SQSUB: return fastEmit_AArch64ISD_SQSUB_rr(VT, RetVT, Op0, Op1);
14206 case AArch64ISD::STRICT_FCMP: return fastEmit_AArch64ISD_STRICT_FCMP_rr(VT, RetVT, Op0, Op1);
14207 case AArch64ISD::STRICT_FCMPE: return fastEmit_AArch64ISD_STRICT_FCMPE_rr(VT, RetVT, Op0, Op1);
14208 case AArch64ISD::TBL: return fastEmit_AArch64ISD_TBL_rr(VT, RetVT, Op0, Op1);
14209 case AArch64ISD::TRN1: return fastEmit_AArch64ISD_TRN1_rr(VT, RetVT, Op0, Op1);
14210 case AArch64ISD::TRN2: return fastEmit_AArch64ISD_TRN2_rr(VT, RetVT, Op0, Op1);
14211 case AArch64ISD::UMULL: return fastEmit_AArch64ISD_UMULL_rr(VT, RetVT, Op0, Op1);
14212 case AArch64ISD::UQADD: return fastEmit_AArch64ISD_UQADD_rr(VT, RetVT, Op0, Op1);
14213 case AArch64ISD::UQRSHL: return fastEmit_AArch64ISD_UQRSHL_rr(VT, RetVT, Op0, Op1);
14214 case AArch64ISD::UQSHL: return fastEmit_AArch64ISD_UQSHL_rr(VT, RetVT, Op0, Op1);
14215 case AArch64ISD::UQSUB: return fastEmit_AArch64ISD_UQSUB_rr(VT, RetVT, Op0, Op1);
14216 case AArch64ISD::UZP1: return fastEmit_AArch64ISD_UZP1_rr(VT, RetVT, Op0, Op1);
14217 case AArch64ISD::UZP2: return fastEmit_AArch64ISD_UZP2_rr(VT, RetVT, Op0, Op1);
14218 case AArch64ISD::ZIP1: return fastEmit_AArch64ISD_ZIP1_rr(VT, RetVT, Op0, Op1);
14219 case AArch64ISD::ZIP2: return fastEmit_AArch64ISD_ZIP2_rr(VT, RetVT, Op0, Op1);
14220 case ISD::ABDS: return fastEmit_ISD_ABDS_rr(VT, RetVT, Op0, Op1);
14221 case ISD::ABDU: return fastEmit_ISD_ABDU_rr(VT, RetVT, Op0, Op1);
14222 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
14223 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
14224 case ISD::AVGCEILS: return fastEmit_ISD_AVGCEILS_rr(VT, RetVT, Op0, Op1);
14225 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
14226 case ISD::AVGFLOORS: return fastEmit_ISD_AVGFLOORS_rr(VT, RetVT, Op0, Op1);
14227 case ISD::AVGFLOORU: return fastEmit_ISD_AVGFLOORU_rr(VT, RetVT, Op0, Op1);
14228 case ISD::CLMUL: return fastEmit_ISD_CLMUL_rr(VT, RetVT, Op0, Op1);
14229 case ISD::CONCAT_VECTORS: return fastEmit_ISD_CONCAT_VECTORS_rr(VT, RetVT, Op0, Op1);
14230 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
14231 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
14232 case ISD::FMAXIMUM: return fastEmit_ISD_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
14233 case ISD::FMAXNUM: return fastEmit_ISD_FMAXNUM_rr(VT, RetVT, Op0, Op1);
14234 case ISD::FMAXNUM_IEEE: return fastEmit_ISD_FMAXNUM_IEEE_rr(VT, RetVT, Op0, Op1);
14235 case ISD::FMINIMUM: return fastEmit_ISD_FMINIMUM_rr(VT, RetVT, Op0, Op1);
14236 case ISD::FMINNUM: return fastEmit_ISD_FMINNUM_rr(VT, RetVT, Op0, Op1);
14237 case ISD::FMINNUM_IEEE: return fastEmit_ISD_FMINNUM_IEEE_rr(VT, RetVT, Op0, Op1);
14238 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
14239 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
14240 case ISD::GET_ACTIVE_LANE_MASK: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_rr(VT, RetVT, Op0, Op1);
14241 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
14242 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
14243 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
14244 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
14245 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
14246 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
14247 case ISD::SDIV: return fastEmit_ISD_SDIV_rr(VT, RetVT, Op0, Op1);
14248 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
14249 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
14250 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
14251 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
14252 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
14253 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
14254 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1);
14255 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1);
14256 case ISD::STRICT_FMAXIMUM: return fastEmit_ISD_STRICT_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
14257 case ISD::STRICT_FMAXNUM: return fastEmit_ISD_STRICT_FMAXNUM_rr(VT, RetVT, Op0, Op1);
14258 case ISD::STRICT_FMINIMUM: return fastEmit_ISD_STRICT_FMINIMUM_rr(VT, RetVT, Op0, Op1);
14259 case ISD::STRICT_FMINNUM: return fastEmit_ISD_STRICT_FMINNUM_rr(VT, RetVT, Op0, Op1);
14260 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1);
14261 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1);
14262 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
14263 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
14264 case ISD::UDIV: return fastEmit_ISD_UDIV_rr(VT, RetVT, Op0, Op1);
14265 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
14266 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
14267 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
14268 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
14269 default: return Register();
14270 }
14271}
14272
14273// FastEmit functions for AArch64ISD::DUPLANE64.
14274
14275Register fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14276 if (RetVT.SimpleTy != MVT::v2i64)
14277 return Register();
14278 if ((Subtarget->isNeonAvailable())) {
14279 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i64lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14280 }
14281 return Register();
14282}
14283
14284Register fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14285 if (RetVT.SimpleTy != MVT::v2f64)
14286 return Register();
14287 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i64lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14288}
14289
14290Register fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14291 switch (VT.SimpleTy) {
14292 case MVT::v2i64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14293 case MVT::v2f64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14294 default: return Register();
14295 }
14296}
14297
14298// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14299
14300Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14301 if (RetVT.SimpleTy != MVT::i64)
14302 return Register();
14303 if ((Subtarget->isNeonAvailable())) {
14304 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi64, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14305 }
14306 return Register();
14307}
14308
14309Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14310 if (RetVT.SimpleTy != MVT::f64)
14311 return Register();
14312 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi64, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14313}
14314
14315Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14316 switch (VT.SimpleTy) {
14317 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14318 case MVT::v2f64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14319 default: return Register();
14320 }
14321}
14322
14323// Top-level FastEmit function.
14324
14325Register fastEmit_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14326 switch (Opcode) {
14327 case AArch64ISD::DUPLANE64: return fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1);
14328 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1);
14329 default: return Register();
14330 }
14331}
14332
14333// FastEmit functions for AArch64ISD::DUPLANE32.
14334
14335Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14336 if ((Subtarget->isNeonAvailable())) {
14337 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i32lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14338 }
14339 return Register();
14340}
14341
14342Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14343 if ((Subtarget->isNeonAvailable())) {
14344 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i32lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14345 }
14346 return Register();
14347}
14348
14349Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14350switch (RetVT.SimpleTy) {
14351 case MVT::v2i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Op0, imm1);
14352 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Op0, imm1);
14353 default: return Register();
14354}
14355}
14356
14357Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14358 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i32lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14359}
14360
14361Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14362 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i32lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14363}
14364
14365Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14366switch (RetVT.SimpleTy) {
14367 case MVT::v2f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Op0, imm1);
14368 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Op0, imm1);
14369 default: return Register();
14370}
14371}
14372
14373Register fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14374 switch (VT.SimpleTy) {
14375 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14376 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14377 default: return Register();
14378 }
14379}
14380
14381// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14382
14383Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14384 if (RetVT.SimpleTy != MVT::i32)
14385 return Register();
14386 if ((Subtarget->isNeonAvailable())) {
14387 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi32, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14388 }
14389 return Register();
14390}
14391
14392Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14393 if (RetVT.SimpleTy != MVT::f32)
14394 return Register();
14395 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi32, RC: &AArch64::FPR32RegClass, Op0, Imm: imm1);
14396}
14397
14398Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14399 switch (VT.SimpleTy) {
14400 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14401 case MVT::v4f32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14402 default: return Register();
14403 }
14404}
14405
14406// Top-level FastEmit function.
14407
14408Register fastEmit_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14409 switch (Opcode) {
14410 case AArch64ISD::DUPLANE32: return fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1);
14411 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1);
14412 default: return Register();
14413 }
14414}
14415
14416// FastEmit functions for AArch64ISD::DUPLANE16.
14417
14418Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14419 if ((Subtarget->isNeonAvailable())) {
14420 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14421 }
14422 return Register();
14423}
14424
14425Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14426 if ((Subtarget->isNeonAvailable())) {
14427 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14428 }
14429 return Register();
14430}
14431
14432Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14433switch (RetVT.SimpleTy) {
14434 case MVT::v4i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Op0, imm1);
14435 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Op0, imm1);
14436 default: return Register();
14437}
14438}
14439
14440Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14441 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14442}
14443
14444Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14445 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14446}
14447
14448Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14449switch (RetVT.SimpleTy) {
14450 case MVT::v4f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Op0, imm1);
14451 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Op0, imm1);
14452 default: return Register();
14453}
14454}
14455
14456Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14457 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14458}
14459
14460Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14461 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14462}
14463
14464Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14465switch (RetVT.SimpleTy) {
14466 case MVT::v4bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(Op0, imm1);
14467 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(Op0, imm1);
14468 default: return Register();
14469}
14470}
14471
14472Register fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14473 switch (VT.SimpleTy) {
14474 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14475 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14476 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14477 default: return Register();
14478 }
14479}
14480
14481// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14482
14483Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14484 if (RetVT.SimpleTy != MVT::i32)
14485 return Register();
14486 if ((Subtarget->isNeonAvailable())) {
14487 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi16, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14488 }
14489 return Register();
14490}
14491
14492Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14493 if (RetVT.SimpleTy != MVT::f16)
14494 return Register();
14495 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi16, RC: &AArch64::FPR16RegClass, Op0, Imm: imm1);
14496}
14497
14498Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14499 if (RetVT.SimpleTy != MVT::bf16)
14500 return Register();
14501 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi16, RC: &AArch64::FPR16RegClass, Op0, Imm: imm1);
14502}
14503
14504Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14505 switch (VT.SimpleTy) {
14506 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14507 case MVT::v8f16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14508 case MVT::v8bf16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14509 default: return Register();
14510 }
14511}
14512
14513// Top-level FastEmit function.
14514
14515Register fastEmit_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14516 switch (Opcode) {
14517 case AArch64ISD::DUPLANE16: return fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1);
14518 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1);
14519 default: return Register();
14520 }
14521}
14522
14523// FastEmit functions for AArch64ISD::DUPLANE8.
14524
14525Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Register Op0, uint64_t imm1) {
14526 if ((Subtarget->isNeonAvailable())) {
14527 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i8lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14528 }
14529 return Register();
14530}
14531
14532Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Register Op0, uint64_t imm1) {
14533 if ((Subtarget->isNeonAvailable())) {
14534 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv16i8lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14535 }
14536 return Register();
14537}
14538
14539Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, Register Op0, uint64_t imm1) {
14540switch (RetVT.SimpleTy) {
14541 case MVT::v8i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Op0, imm1);
14542 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Op0, imm1);
14543 default: return Register();
14544}
14545}
14546
14547Register fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14548 switch (VT.SimpleTy) {
14549 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1);
14550 default: return Register();
14551 }
14552}
14553
14554// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14555
14556Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, Register Op0, uint64_t imm1) {
14557 if (RetVT.SimpleTy != MVT::i32)
14558 return Register();
14559 if ((Subtarget->isNeonAvailable())) {
14560 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi8, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14561 }
14562 return Register();
14563}
14564
14565Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14566 switch (VT.SimpleTy) {
14567 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1);
14568 default: return Register();
14569 }
14570}
14571
14572// Top-level FastEmit function.
14573
14574Register fastEmit_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14575 switch (Opcode) {
14576 case AArch64ISD::DUPLANE8: return fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1);
14577 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1);
14578 default: return Register();
14579 }
14580}
14581
14582// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14583
14584Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14585 if (RetVT.SimpleTy != MVT::i32)
14586 return Register();
14587 if ((Subtarget->hasNEON())) {
14588 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi8_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14589 }
14590 return Register();
14591}
14592
14593Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14594 if (RetVT.SimpleTy != MVT::i32)
14595 return Register();
14596 if ((Subtarget->hasNEON())) {
14597 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi16_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14598 }
14599 return Register();
14600}
14601
14602Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14603 if (RetVT.SimpleTy != MVT::i32)
14604 return Register();
14605 if ((Subtarget->hasNEON())) {
14606 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi32_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14607 }
14608 return Register();
14609}
14610
14611Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14612 if (RetVT.SimpleTy != MVT::i64)
14613 return Register();
14614 if ((Subtarget->hasNEON())) {
14615 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi64_idx0, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14616 }
14617 return Register();
14618}
14619
14620Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14621 switch (VT.SimpleTy) {
14622 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14623 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14624 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14625 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14626 default: return Register();
14627 }
14628}
14629
14630// Top-level FastEmit function.
14631
14632Register fastEmit_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14633 switch (Opcode) {
14634 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(VT, RetVT, Op0, imm1);
14635 default: return Register();
14636 }
14637}
14638
14639// FastEmit functions for ISD::SMAX.
14640
14641Register fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14642 if (RetVT.SimpleTy != MVT::i32)
14643 return Register();
14644 if ((Subtarget->hasCSSC())) {
14645 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMAXWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14646 }
14647 return Register();
14648}
14649
14650Register fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14651 switch (VT.SimpleTy) {
14652 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1);
14653 default: return Register();
14654 }
14655}
14656
14657// FastEmit functions for ISD::SMIN.
14658
14659Register fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14660 if (RetVT.SimpleTy != MVT::i32)
14661 return Register();
14662 if ((Subtarget->hasCSSC())) {
14663 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMINWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14664 }
14665 return Register();
14666}
14667
14668Register fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14669 switch (VT.SimpleTy) {
14670 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1);
14671 default: return Register();
14672 }
14673}
14674
14675// Top-level FastEmit function.
14676
14677Register fastEmit_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14678 switch (Opcode) {
14679 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1);
14680 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1);
14681 default: return Register();
14682 }
14683}
14684
14685// FastEmit functions for ISD::SMAX.
14686
14687Register fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14688 if (RetVT.SimpleTy != MVT::i64)
14689 return Register();
14690 if ((Subtarget->hasCSSC())) {
14691 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMAXXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14692 }
14693 return Register();
14694}
14695
14696Register fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14697 switch (VT.SimpleTy) {
14698 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1);
14699 default: return Register();
14700 }
14701}
14702
14703// FastEmit functions for ISD::SMIN.
14704
14705Register fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14706 if (RetVT.SimpleTy != MVT::i64)
14707 return Register();
14708 if ((Subtarget->hasCSSC())) {
14709 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMINXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14710 }
14711 return Register();
14712}
14713
14714Register fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14715 switch (VT.SimpleTy) {
14716 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1);
14717 default: return Register();
14718 }
14719}
14720
14721// Top-level FastEmit function.
14722
14723Register fastEmit_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14724 switch (Opcode) {
14725 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1);
14726 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1);
14727 default: return Register();
14728 }
14729}
14730
14731// FastEmit functions for ISD::UMAX.
14732
14733Register fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14734 if (RetVT.SimpleTy != MVT::i32)
14735 return Register();
14736 if ((Subtarget->hasCSSC())) {
14737 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMAXWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14738 }
14739 return Register();
14740}
14741
14742Register fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14743 switch (VT.SimpleTy) {
14744 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1);
14745 default: return Register();
14746 }
14747}
14748
14749// FastEmit functions for ISD::UMIN.
14750
14751Register fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14752 if (RetVT.SimpleTy != MVT::i32)
14753 return Register();
14754 if ((Subtarget->hasCSSC())) {
14755 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMINWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14756 }
14757 return Register();
14758}
14759
14760Register fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14761 switch (VT.SimpleTy) {
14762 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1);
14763 default: return Register();
14764 }
14765}
14766
14767// Top-level FastEmit function.
14768
14769Register fastEmit_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14770 switch (Opcode) {
14771 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1);
14772 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1);
14773 default: return Register();
14774 }
14775}
14776
14777// FastEmit functions for ISD::UMAX.
14778
14779Register fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14780 if (RetVT.SimpleTy != MVT::i64)
14781 return Register();
14782 if ((Subtarget->hasCSSC())) {
14783 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMAXXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14784 }
14785 return Register();
14786}
14787
14788Register fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14789 switch (VT.SimpleTy) {
14790 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1);
14791 default: return Register();
14792 }
14793}
14794
14795// FastEmit functions for ISD::UMIN.
14796
14797Register fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14798 if (RetVT.SimpleTy != MVT::i64)
14799 return Register();
14800 if ((Subtarget->hasCSSC())) {
14801 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMINXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14802 }
14803 return Register();
14804}
14805
14806Register fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14807 switch (VT.SimpleTy) {
14808 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1);
14809 default: return Register();
14810 }
14811}
14812
14813// Top-level FastEmit function.
14814
14815Register fastEmit_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14816 switch (Opcode) {
14817 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1);
14818 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1);
14819 default: return Register();
14820 }
14821}
14822
14823// FastEmit functions for ISD::Constant.
14824
14825Register fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
14826 if (RetVT.SimpleTy != MVT::i32)
14827 return Register();
14828 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVi32imm, RC: &AArch64::GPR32RegClass, Imm: imm0);
14829}
14830
14831Register fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) {
14832 if (RetVT.SimpleTy != MVT::i64)
14833 return Register();
14834 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVi64imm, RC: &AArch64::GPR64RegClass, Imm: imm0);
14835}
14836
14837Register fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
14838 switch (VT.SimpleTy) {
14839 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
14840 case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0);
14841 default: return Register();
14842 }
14843}
14844
14845// Top-level FastEmit function.
14846
14847Register fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
14848 if (VT == MVT::i32 && Predicate_imm0_255(Imm: imm0))
14849 if (Register Reg = fastEmit_i_Predicate_imm0_255(VT, RetVT, Opcode, imm0))
14850 return Reg;
14851
14852 if (VT == MVT::i32 && Predicate_simm6_32b(Imm: imm0))
14853 if (Register Reg = fastEmit_i_Predicate_simm6_32b(VT, RetVT, Opcode, imm0))
14854 return Reg;
14855
14856 switch (Opcode) {
14857 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
14858 default: return Register();
14859 }
14860}
14861
14862// FastEmit functions for AArch64ISD::FMOV.
14863
14864Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(uint64_t imm0) {
14865 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
14866 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv4f16_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
14867 }
14868 return Register();
14869}
14870
14871Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(uint64_t imm0) {
14872 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
14873 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv8f16_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14874 }
14875 return Register();
14876}
14877
14878Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(uint64_t imm0) {
14879 if ((Subtarget->isNeonAvailable())) {
14880 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv2f32_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
14881 }
14882 return Register();
14883}
14884
14885Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(uint64_t imm0) {
14886 if ((Subtarget->isNeonAvailable())) {
14887 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv4f32_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14888 }
14889 return Register();
14890}
14891
14892Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(uint64_t imm0) {
14893 if ((Subtarget->isNeonAvailable())) {
14894 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv2f64_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14895 }
14896 return Register();
14897}
14898
14899Register fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
14900switch (RetVT.SimpleTy) {
14901 case MVT::v4f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(imm0);
14902 case MVT::v8f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(imm0);
14903 case MVT::v2f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(imm0);
14904 case MVT::v4f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(imm0);
14905 case MVT::v2f64: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(imm0);
14906 default: return Register();
14907}
14908}
14909
14910Register fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
14911 switch (VT.SimpleTy) {
14912 case MVT::i32: return fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
14913 default: return Register();
14914 }
14915}
14916
14917// FastEmit functions for AArch64ISD::MOVI.
14918
14919Register fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(uint64_t imm0) {
14920 if ((Subtarget->isNeonAvailable())) {
14921 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv8b_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
14922 }
14923 return Register();
14924}
14925
14926Register fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(uint64_t imm0) {
14927 if ((Subtarget->isNeonAvailable())) {
14928 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv16b_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14929 }
14930 return Register();
14931}
14932
14933Register fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
14934switch (RetVT.SimpleTy) {
14935 case MVT::v8i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(imm0);
14936 case MVT::v16i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(imm0);
14937 default: return Register();
14938}
14939}
14940
14941Register fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
14942 switch (VT.SimpleTy) {
14943 case MVT::i32: return fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
14944 default: return Register();
14945 }
14946}
14947
14948// FastEmit functions for AArch64ISD::MOVIedit.
14949
14950Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(uint64_t imm0) {
14951 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVID, RC: &AArch64::FPR64RegClass, Imm: imm0);
14952}
14953
14954Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(uint64_t imm0) {
14955 if ((Subtarget->isNeonAvailable())) {
14956 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv2d_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14957 }
14958 return Register();
14959}
14960
14961Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
14962switch (RetVT.SimpleTy) {
14963 case MVT::f64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(imm0);
14964 case MVT::v2i64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(imm0);
14965 default: return Register();
14966}
14967}
14968
14969Register fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
14970 switch (VT.SimpleTy) {
14971 case MVT::i32: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
14972 default: return Register();
14973 }
14974}
14975
14976// Top-level FastEmit function.
14977
14978Register fastEmit_i_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
14979 switch (Opcode) {
14980 case AArch64ISD::FMOV: return fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(VT, RetVT, imm0);
14981 case AArch64ISD::MOVI: return fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(VT, RetVT, imm0);
14982 case AArch64ISD::MOVIedit: return fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(VT, RetVT, imm0);
14983 default: return Register();
14984 }
14985}
14986
14987// FastEmit functions for AArch64ISD::RDSVL.
14988
14989Register fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(MVT RetVT, uint64_t imm0) {
14990 if (RetVT.SimpleTy != MVT::i64)
14991 return Register();
14992 if ((Subtarget->hasSME())) {
14993 return fastEmitInst_i(MachineInstOpcode: AArch64::RDSVLI_XI, RC: &AArch64::GPR64RegClass, Imm: imm0);
14994 }
14995 return Register();
14996}
14997
14998Register fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(MVT VT, MVT RetVT, uint64_t imm0) {
14999 switch (VT.SimpleTy) {
15000 case MVT::i32: return fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(RetVT, imm0);
15001 default: return Register();
15002 }
15003}
15004
15005// Top-level FastEmit function.
15006
15007Register fastEmit_i_Predicate_simm6_32b(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
15008 switch (Opcode) {
15009 case AArch64ISD::RDSVL: return fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(VT, RetVT, imm0);
15010 default: return Register();
15011 }
15012}
15013
15014