1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the AArch64 target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_imm0_31(int64_t Imm) {
12
13 return ((uint64_t)Imm) < 32;
14
15}
16static bool Predicate_imm0_63(int64_t Imm) {
17
18 return ((uint64_t)Imm) < 64;
19
20}
21static bool Predicate_imm32_0_31(int64_t Imm) {
22
23 return ((uint64_t)Imm) < 32;
24
25}
26static bool Predicate_tbz_imm0_31_diag(int64_t Imm) {
27
28 return (((uint32_t)Imm) < 32);
29
30}
31static bool Predicate_tbz_imm32_63(int64_t Imm) {
32
33 return (((uint32_t)Imm) > 31) && (((uint32_t)Imm) < 64);
34
35}
36static bool Predicate_VectorIndexD(int64_t Imm) {
37 return ((uint64_t)Imm) < 2;
38}
39static bool Predicate_VectorIndexS(int64_t Imm) {
40 return ((uint64_t)Imm) < 4;
41}
42static bool Predicate_VectorIndexH(int64_t Imm) {
43 return ((uint64_t)Imm) < 8;
44}
45static bool Predicate_VectorIndexB(int64_t Imm) {
46 return ((uint64_t)Imm) < 16;
47}
48static bool Predicate_VectorIndex0(int64_t Imm) {
49 return ((uint64_t)Imm) == 0;
50}
51static bool Predicate_imm0_255(int64_t Imm) {
52
53 return ((uint32_t)Imm) < 256;
54
55}
56static bool Predicate_simm8_32b(int64_t Imm) {
57 return Imm >= -128 && Imm < 128;
58}
59static bool Predicate_simm8_64b(int64_t Imm) {
60 return Imm >= -128 && Imm < 128;
61}
62static bool Predicate_uimm8_32b(int64_t Imm) {
63 return Imm >= 0 && Imm < 256;
64}
65static bool Predicate_uimm8_64b(int64_t Imm) {
66 return Imm >= 0 && Imm < 256;
67}
68static bool Predicate_simm6_32b(int64_t Imm) {
69 return Imm >= -32 && Imm < 32;
70}
71
72
73// FastEmit functions for AArch64ISD::ENTRY_PSTATE_SM.
74
75Register fastEmit_AArch64ISD_ENTRY_PSTATE_SM_MVT_i64_(MVT RetVT) {
76 if (RetVT.SimpleTy != MVT::i64)
77 return Register();
78 return fastEmitInst_(MachineInstOpcode: AArch64::EntryPStateSM, RC: &AArch64::GPR64RegClass);
79}
80
81Register fastEmit_AArch64ISD_ENTRY_PSTATE_SM_(MVT VT, MVT RetVT) {
82 switch (VT.SimpleTy) {
83 case MVT::i64: return fastEmit_AArch64ISD_ENTRY_PSTATE_SM_MVT_i64_(RetVT);
84 default: return Register();
85 }
86}
87
88// FastEmit functions for AArch64ISD::THREAD_POINTER.
89
90Register fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(MVT RetVT) {
91 if (RetVT.SimpleTy != MVT::i64)
92 return Register();
93 return fastEmitInst_(MachineInstOpcode: AArch64::MOVbaseTLS, RC: &AArch64::GPR64RegClass);
94}
95
96Register fastEmit_AArch64ISD_THREAD_POINTER_(MVT VT, MVT RetVT) {
97 switch (VT.SimpleTy) {
98 case MVT::i64: return fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(RetVT);
99 default: return Register();
100 }
101}
102
103// Top-level FastEmit function.
104
105Register fastEmit_(MVT VT, MVT RetVT, unsigned Opcode) override {
106 switch (Opcode) {
107 case AArch64ISD::ENTRY_PSTATE_SM: return fastEmit_AArch64ISD_ENTRY_PSTATE_SM_(VT, RetVT);
108 case AArch64ISD::THREAD_POINTER: return fastEmit_AArch64ISD_THREAD_POINTER_(VT, RetVT);
109 default: return Register();
110 }
111}
112
113// FastEmit functions for AArch64ISD::CALL.
114
115Register fastEmit_AArch64ISD_CALL_MVT_i64_r(MVT RetVT, Register Op0) {
116 if (RetVT.SimpleTy != MVT::isVoid)
117 return Register();
118 if (( MF->getSubtarget<AArch64Subtarget>().hardenSlsBlr() )) {
119 return fastEmitInst_r(MachineInstOpcode: AArch64::BLRNoIP, RC: &AArch64::GPR64noipRegClass, Op0);
120 }
121 if (( !MF->getSubtarget<AArch64Subtarget>().hardenSlsBlr() )) {
122 return fastEmitInst_r(MachineInstOpcode: AArch64::BLR, RC: &AArch64::GPR64RegClass, Op0);
123 }
124 return Register();
125}
126
127Register fastEmit_AArch64ISD_CALL_r(MVT VT, MVT RetVT, Register Op0) {
128 switch (VT.SimpleTy) {
129 case MVT::i64: return fastEmit_AArch64ISD_CALL_MVT_i64_r(RetVT, Op0);
130 default: return Register();
131 }
132}
133
134// FastEmit functions for AArch64ISD::COALESCER_BARRIER.
135
136Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_bf16_r(MVT RetVT, Register Op0) {
137 if (RetVT.SimpleTy != MVT::bf16)
138 return Register();
139 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR16, RC: &AArch64::FPR16RegClass, Op0);
140}
141
142Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f16_r(MVT RetVT, Register Op0) {
143 if (RetVT.SimpleTy != MVT::f16)
144 return Register();
145 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR16, RC: &AArch64::FPR16RegClass, Op0);
146}
147
148Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f32_r(MVT RetVT, Register Op0) {
149 if (RetVT.SimpleTy != MVT::f32)
150 return Register();
151 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR32, RC: &AArch64::FPR32RegClass, Op0);
152}
153
154Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f64_r(MVT RetVT, Register Op0) {
155 if (RetVT.SimpleTy != MVT::f64)
156 return Register();
157 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
158}
159
160Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f128_r(MVT RetVT, Register Op0) {
161 if (RetVT.SimpleTy != MVT::f128)
162 return Register();
163 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
164}
165
166Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i8_r(MVT RetVT, Register Op0) {
167 if (RetVT.SimpleTy != MVT::v8i8)
168 return Register();
169 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
170}
171
172Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v16i8_r(MVT RetVT, Register Op0) {
173 if (RetVT.SimpleTy != MVT::v16i8)
174 return Register();
175 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
176}
177
178Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i16_r(MVT RetVT, Register Op0) {
179 if (RetVT.SimpleTy != MVT::v4i16)
180 return Register();
181 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
182}
183
184Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i16_r(MVT RetVT, Register Op0) {
185 if (RetVT.SimpleTy != MVT::v8i16)
186 return Register();
187 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
188}
189
190Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i32_r(MVT RetVT, Register Op0) {
191 if (RetVT.SimpleTy != MVT::v2i32)
192 return Register();
193 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
194}
195
196Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i32_r(MVT RetVT, Register Op0) {
197 if (RetVT.SimpleTy != MVT::v4i32)
198 return Register();
199 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
200}
201
202Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1i64_r(MVT RetVT, Register Op0) {
203 if (RetVT.SimpleTy != MVT::v1i64)
204 return Register();
205 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
206}
207
208Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i64_r(MVT RetVT, Register Op0) {
209 if (RetVT.SimpleTy != MVT::v2i64)
210 return Register();
211 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
212}
213
214Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f16_r(MVT RetVT, Register Op0) {
215 if (RetVT.SimpleTy != MVT::v4f16)
216 return Register();
217 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
218}
219
220Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8f16_r(MVT RetVT, Register Op0) {
221 if (RetVT.SimpleTy != MVT::v8f16)
222 return Register();
223 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
224}
225
226Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4bf16_r(MVT RetVT, Register Op0) {
227 if (RetVT.SimpleTy != MVT::v4bf16)
228 return Register();
229 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
230}
231
232Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8bf16_r(MVT RetVT, Register Op0) {
233 if (RetVT.SimpleTy != MVT::v8bf16)
234 return Register();
235 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
236}
237
238Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f32_r(MVT RetVT, Register Op0) {
239 if (RetVT.SimpleTy != MVT::v2f32)
240 return Register();
241 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
242}
243
244Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f32_r(MVT RetVT, Register Op0) {
245 if (RetVT.SimpleTy != MVT::v4f32)
246 return Register();
247 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
248}
249
250Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1f64_r(MVT RetVT, Register Op0) {
251 if (RetVT.SimpleTy != MVT::v1f64)
252 return Register();
253 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
254}
255
256Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f64_r(MVT RetVT, Register Op0) {
257 if (RetVT.SimpleTy != MVT::v2f64)
258 return Register();
259 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
260}
261
262Register fastEmit_AArch64ISD_COALESCER_BARRIER_r(MVT VT, MVT RetVT, Register Op0) {
263 switch (VT.SimpleTy) {
264 case MVT::bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_bf16_r(RetVT, Op0);
265 case MVT::f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f16_r(RetVT, Op0);
266 case MVT::f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f32_r(RetVT, Op0);
267 case MVT::f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f64_r(RetVT, Op0);
268 case MVT::f128: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f128_r(RetVT, Op0);
269 case MVT::v8i8: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i8_r(RetVT, Op0);
270 case MVT::v16i8: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v16i8_r(RetVT, Op0);
271 case MVT::v4i16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i16_r(RetVT, Op0);
272 case MVT::v8i16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i16_r(RetVT, Op0);
273 case MVT::v2i32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i32_r(RetVT, Op0);
274 case MVT::v4i32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i32_r(RetVT, Op0);
275 case MVT::v1i64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1i64_r(RetVT, Op0);
276 case MVT::v2i64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i64_r(RetVT, Op0);
277 case MVT::v4f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f16_r(RetVT, Op0);
278 case MVT::v8f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8f16_r(RetVT, Op0);
279 case MVT::v4bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4bf16_r(RetVT, Op0);
280 case MVT::v8bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8bf16_r(RetVT, Op0);
281 case MVT::v2f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f32_r(RetVT, Op0);
282 case MVT::v4f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f32_r(RetVT, Op0);
283 case MVT::v1f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1f64_r(RetVT, Op0);
284 case MVT::v2f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f64_r(RetVT, Op0);
285 default: return Register();
286 }
287}
288
289// FastEmit functions for AArch64ISD::DUP.
290
291Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Register Op0) {
292 if ((Subtarget->isNeonAvailable())) {
293 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv8i8gpr, RC: &AArch64::FPR64RegClass, Op0);
294 }
295 return Register();
296}
297
298Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Register Op0) {
299 if ((Subtarget->isNeonAvailable())) {
300 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv16i8gpr, RC: &AArch64::FPR128RegClass, Op0);
301 }
302 return Register();
303}
304
305Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Register Op0) {
306 if ((Subtarget->isNeonAvailable())) {
307 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv4i16gpr, RC: &AArch64::FPR64RegClass, Op0);
308 }
309 return Register();
310}
311
312Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Register Op0) {
313 if ((Subtarget->isNeonAvailable())) {
314 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv8i16gpr, RC: &AArch64::FPR128RegClass, Op0);
315 }
316 return Register();
317}
318
319Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Register Op0) {
320 if ((Subtarget->isNeonAvailable())) {
321 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv2i32gpr, RC: &AArch64::FPR64RegClass, Op0);
322 }
323 return Register();
324}
325
326Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Register Op0) {
327 if ((Subtarget->isNeonAvailable())) {
328 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv4i32gpr, RC: &AArch64::FPR128RegClass, Op0);
329 }
330 return Register();
331}
332
333Register fastEmit_AArch64ISD_DUP_MVT_i32_r(MVT RetVT, Register Op0) {
334switch (RetVT.SimpleTy) {
335 case MVT::v8i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Op0);
336 case MVT::v16i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Op0);
337 case MVT::v4i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Op0);
338 case MVT::v8i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Op0);
339 case MVT::v2i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Op0);
340 case MVT::v4i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Op0);
341 default: return Register();
342}
343}
344
345Register fastEmit_AArch64ISD_DUP_MVT_i64_r(MVT RetVT, Register Op0) {
346 if (RetVT.SimpleTy != MVT::v2i64)
347 return Register();
348 if ((Subtarget->isNeonAvailable())) {
349 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv2i64gpr, RC: &AArch64::FPR128RegClass, Op0);
350 }
351 return Register();
352}
353
354Register fastEmit_AArch64ISD_DUP_r(MVT VT, MVT RetVT, Register Op0) {
355 switch (VT.SimpleTy) {
356 case MVT::i32: return fastEmit_AArch64ISD_DUP_MVT_i32_r(RetVT, Op0);
357 case MVT::i64: return fastEmit_AArch64ISD_DUP_MVT_i64_r(RetVT, Op0);
358 default: return Register();
359 }
360}
361
362// FastEmit functions for AArch64ISD::FCVTXN.
363
364Register fastEmit_AArch64ISD_FCVTXN_MVT_f64_r(MVT RetVT, Register Op0) {
365 if (RetVT.SimpleTy != MVT::f32)
366 return Register();
367 if ((Subtarget->isNeonAvailable())) {
368 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTXNv1i64, RC: &AArch64::FPR32RegClass, Op0);
369 }
370 return Register();
371}
372
373Register fastEmit_AArch64ISD_FCVTXN_MVT_v2f64_r(MVT RetVT, Register Op0) {
374 if (RetVT.SimpleTy != MVT::v2f32)
375 return Register();
376 if ((Subtarget->isNeonAvailable())) {
377 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTXNv2f32, RC: &AArch64::FPR64RegClass, Op0);
378 }
379 return Register();
380}
381
382Register fastEmit_AArch64ISD_FCVTXN_r(MVT VT, MVT RetVT, Register Op0) {
383 switch (VT.SimpleTy) {
384 case MVT::f64: return fastEmit_AArch64ISD_FCVTXN_MVT_f64_r(RetVT, Op0);
385 case MVT::v2f64: return fastEmit_AArch64ISD_FCVTXN_MVT_v2f64_r(RetVT, Op0);
386 default: return Register();
387 }
388}
389
390// FastEmit functions for AArch64ISD::FRECPE.
391
392Register fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(MVT RetVT, Register Op0) {
393 if (RetVT.SimpleTy != MVT::v2f32)
394 return Register();
395 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv2f32, RC: &AArch64::FPR64RegClass, Op0);
396}
397
398Register fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(MVT RetVT, Register Op0) {
399 if (RetVT.SimpleTy != MVT::v4f32)
400 return Register();
401 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv4f32, RC: &AArch64::FPR128RegClass, Op0);
402}
403
404Register fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(MVT RetVT, Register Op0) {
405 if (RetVT.SimpleTy != MVT::v2f64)
406 return Register();
407 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv2f64, RC: &AArch64::FPR128RegClass, Op0);
408}
409
410Register fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
411 if (RetVT.SimpleTy != MVT::nxv8f16)
412 return Register();
413 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
414 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
415 }
416 return Register();
417}
418
419Register fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
420 if (RetVT.SimpleTy != MVT::nxv4f32)
421 return Register();
422 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
423 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
424 }
425 return Register();
426}
427
428Register fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
429 if (RetVT.SimpleTy != MVT::nxv2f64)
430 return Register();
431 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
432 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
433 }
434 return Register();
435}
436
437Register fastEmit_AArch64ISD_FRECPE_r(MVT VT, MVT RetVT, Register Op0) {
438 switch (VT.SimpleTy) {
439 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(RetVT, Op0);
440 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(RetVT, Op0);
441 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(RetVT, Op0);
442 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(RetVT, Op0);
443 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(RetVT, Op0);
444 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(RetVT, Op0);
445 default: return Register();
446 }
447}
448
449// FastEmit functions for AArch64ISD::FRSQRTE.
450
451Register fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(MVT RetVT, Register Op0) {
452 if (RetVT.SimpleTy != MVT::v2f32)
453 return Register();
454 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv2f32, RC: &AArch64::FPR64RegClass, Op0);
455}
456
457Register fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(MVT RetVT, Register Op0) {
458 if (RetVT.SimpleTy != MVT::v4f32)
459 return Register();
460 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv4f32, RC: &AArch64::FPR128RegClass, Op0);
461}
462
463Register fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(MVT RetVT, Register Op0) {
464 if (RetVT.SimpleTy != MVT::v2f64)
465 return Register();
466 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv2f64, RC: &AArch64::FPR128RegClass, Op0);
467}
468
469Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
470 if (RetVT.SimpleTy != MVT::nxv8f16)
471 return Register();
472 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
473 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
474 }
475 return Register();
476}
477
478Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
479 if (RetVT.SimpleTy != MVT::nxv4f32)
480 return Register();
481 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
482 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
483 }
484 return Register();
485}
486
487Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
488 if (RetVT.SimpleTy != MVT::nxv2f64)
489 return Register();
490 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
491 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
492 }
493 return Register();
494}
495
496Register fastEmit_AArch64ISD_FRSQRTE_r(MVT VT, MVT RetVT, Register Op0) {
497 switch (VT.SimpleTy) {
498 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(RetVT, Op0);
499 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(RetVT, Op0);
500 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(RetVT, Op0);
501 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(RetVT, Op0);
502 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(RetVT, Op0);
503 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(RetVT, Op0);
504 default: return Register();
505 }
506}
507
508// FastEmit functions for AArch64ISD::PROBED_ALLOCA.
509
510Register fastEmit_AArch64ISD_PROBED_ALLOCA_MVT_i64_r(MVT RetVT, Register Op0) {
511 if (RetVT.SimpleTy != MVT::isVoid)
512 return Register();
513 return fastEmitInst_r(MachineInstOpcode: AArch64::PROBED_STACKALLOC_DYN, RC: &AArch64::GPR64commonRegClass, Op0);
514}
515
516Register fastEmit_AArch64ISD_PROBED_ALLOCA_r(MVT VT, MVT RetVT, Register Op0) {
517 switch (VT.SimpleTy) {
518 case MVT::i64: return fastEmit_AArch64ISD_PROBED_ALLOCA_MVT_i64_r(RetVT, Op0);
519 default: return Register();
520 }
521}
522
523// FastEmit functions for AArch64ISD::REV16.
524
525Register fastEmit_AArch64ISD_REV16_MVT_i32_r(MVT RetVT, Register Op0) {
526 if (RetVT.SimpleTy != MVT::i32)
527 return Register();
528 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16Wr, RC: &AArch64::GPR32RegClass, Op0);
529}
530
531Register fastEmit_AArch64ISD_REV16_MVT_i64_r(MVT RetVT, Register Op0) {
532 if (RetVT.SimpleTy != MVT::i64)
533 return Register();
534 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16Xr, RC: &AArch64::GPR64RegClass, Op0);
535}
536
537Register fastEmit_AArch64ISD_REV16_r(MVT VT, MVT RetVT, Register Op0) {
538 switch (VT.SimpleTy) {
539 case MVT::i32: return fastEmit_AArch64ISD_REV16_MVT_i32_r(RetVT, Op0);
540 case MVT::i64: return fastEmit_AArch64ISD_REV16_MVT_i64_r(RetVT, Op0);
541 default: return Register();
542 }
543}
544
545// FastEmit functions for AArch64ISD::REV32.
546
547Register fastEmit_AArch64ISD_REV32_MVT_v8i8_r(MVT RetVT, Register Op0) {
548 if (RetVT.SimpleTy != MVT::v8i8)
549 return Register();
550 if ((Subtarget->isNeonAvailable())) {
551 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
552 }
553 return Register();
554}
555
556Register fastEmit_AArch64ISD_REV32_MVT_v16i8_r(MVT RetVT, Register Op0) {
557 if (RetVT.SimpleTy != MVT::v16i8)
558 return Register();
559 if ((Subtarget->isNeonAvailable())) {
560 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
561 }
562 return Register();
563}
564
565Register fastEmit_AArch64ISD_REV32_MVT_v4i16_r(MVT RetVT, Register Op0) {
566 if (RetVT.SimpleTy != MVT::v4i16)
567 return Register();
568 if ((Subtarget->isNeonAvailable())) {
569 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
570 }
571 return Register();
572}
573
574Register fastEmit_AArch64ISD_REV32_MVT_v8i16_r(MVT RetVT, Register Op0) {
575 if (RetVT.SimpleTy != MVT::v8i16)
576 return Register();
577 if ((Subtarget->isNeonAvailable())) {
578 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
579 }
580 return Register();
581}
582
583Register fastEmit_AArch64ISD_REV32_MVT_v4f16_r(MVT RetVT, Register Op0) {
584 if (RetVT.SimpleTy != MVT::v4f16)
585 return Register();
586 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
587}
588
589Register fastEmit_AArch64ISD_REV32_MVT_v8f16_r(MVT RetVT, Register Op0) {
590 if (RetVT.SimpleTy != MVT::v8f16)
591 return Register();
592 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
593}
594
595Register fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(MVT RetVT, Register Op0) {
596 if (RetVT.SimpleTy != MVT::v4bf16)
597 return Register();
598 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
599}
600
601Register fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(MVT RetVT, Register Op0) {
602 if (RetVT.SimpleTy != MVT::v8bf16)
603 return Register();
604 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
605}
606
607Register fastEmit_AArch64ISD_REV32_r(MVT VT, MVT RetVT, Register Op0) {
608 switch (VT.SimpleTy) {
609 case MVT::v8i8: return fastEmit_AArch64ISD_REV32_MVT_v8i8_r(RetVT, Op0);
610 case MVT::v16i8: return fastEmit_AArch64ISD_REV32_MVT_v16i8_r(RetVT, Op0);
611 case MVT::v4i16: return fastEmit_AArch64ISD_REV32_MVT_v4i16_r(RetVT, Op0);
612 case MVT::v8i16: return fastEmit_AArch64ISD_REV32_MVT_v8i16_r(RetVT, Op0);
613 case MVT::v4f16: return fastEmit_AArch64ISD_REV32_MVT_v4f16_r(RetVT, Op0);
614 case MVT::v8f16: return fastEmit_AArch64ISD_REV32_MVT_v8f16_r(RetVT, Op0);
615 case MVT::v4bf16: return fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(RetVT, Op0);
616 case MVT::v8bf16: return fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(RetVT, Op0);
617 default: return Register();
618 }
619}
620
621// FastEmit functions for AArch64ISD::REV64.
622
623Register fastEmit_AArch64ISD_REV64_MVT_v8i8_r(MVT RetVT, Register Op0) {
624 if (RetVT.SimpleTy != MVT::v8i8)
625 return Register();
626 if ((Subtarget->isNeonAvailable())) {
627 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
628 }
629 return Register();
630}
631
632Register fastEmit_AArch64ISD_REV64_MVT_v16i8_r(MVT RetVT, Register Op0) {
633 if (RetVT.SimpleTy != MVT::v16i8)
634 return Register();
635 if ((Subtarget->isNeonAvailable())) {
636 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
637 }
638 return Register();
639}
640
641Register fastEmit_AArch64ISD_REV64_MVT_v4i16_r(MVT RetVT, Register Op0) {
642 if (RetVT.SimpleTy != MVT::v4i16)
643 return Register();
644 if ((Subtarget->isNeonAvailable())) {
645 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
646 }
647 return Register();
648}
649
650Register fastEmit_AArch64ISD_REV64_MVT_v8i16_r(MVT RetVT, Register Op0) {
651 if (RetVT.SimpleTy != MVT::v8i16)
652 return Register();
653 if ((Subtarget->isNeonAvailable())) {
654 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
655 }
656 return Register();
657}
658
659Register fastEmit_AArch64ISD_REV64_MVT_v2i32_r(MVT RetVT, Register Op0) {
660 if (RetVT.SimpleTy != MVT::v2i32)
661 return Register();
662 if ((Subtarget->isNeonAvailable())) {
663 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
664 }
665 return Register();
666}
667
668Register fastEmit_AArch64ISD_REV64_MVT_v4i32_r(MVT RetVT, Register Op0) {
669 if (RetVT.SimpleTy != MVT::v4i32)
670 return Register();
671 if ((Subtarget->isNeonAvailable())) {
672 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
673 }
674 return Register();
675}
676
677Register fastEmit_AArch64ISD_REV64_MVT_v4f16_r(MVT RetVT, Register Op0) {
678 if (RetVT.SimpleTy != MVT::v4f16)
679 return Register();
680 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
681}
682
683Register fastEmit_AArch64ISD_REV64_MVT_v8f16_r(MVT RetVT, Register Op0) {
684 if (RetVT.SimpleTy != MVT::v8f16)
685 return Register();
686 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
687}
688
689Register fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(MVT RetVT, Register Op0) {
690 if (RetVT.SimpleTy != MVT::v4bf16)
691 return Register();
692 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
693}
694
695Register fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(MVT RetVT, Register Op0) {
696 if (RetVT.SimpleTy != MVT::v8bf16)
697 return Register();
698 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
699}
700
701Register fastEmit_AArch64ISD_REV64_MVT_v2f32_r(MVT RetVT, Register Op0) {
702 if (RetVT.SimpleTy != MVT::v2f32)
703 return Register();
704 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
705}
706
707Register fastEmit_AArch64ISD_REV64_MVT_v4f32_r(MVT RetVT, Register Op0) {
708 if (RetVT.SimpleTy != MVT::v4f32)
709 return Register();
710 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
711}
712
713Register fastEmit_AArch64ISD_REV64_r(MVT VT, MVT RetVT, Register Op0) {
714 switch (VT.SimpleTy) {
715 case MVT::v8i8: return fastEmit_AArch64ISD_REV64_MVT_v8i8_r(RetVT, Op0);
716 case MVT::v16i8: return fastEmit_AArch64ISD_REV64_MVT_v16i8_r(RetVT, Op0);
717 case MVT::v4i16: return fastEmit_AArch64ISD_REV64_MVT_v4i16_r(RetVT, Op0);
718 case MVT::v8i16: return fastEmit_AArch64ISD_REV64_MVT_v8i16_r(RetVT, Op0);
719 case MVT::v2i32: return fastEmit_AArch64ISD_REV64_MVT_v2i32_r(RetVT, Op0);
720 case MVT::v4i32: return fastEmit_AArch64ISD_REV64_MVT_v4i32_r(RetVT, Op0);
721 case MVT::v4f16: return fastEmit_AArch64ISD_REV64_MVT_v4f16_r(RetVT, Op0);
722 case MVT::v8f16: return fastEmit_AArch64ISD_REV64_MVT_v8f16_r(RetVT, Op0);
723 case MVT::v4bf16: return fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(RetVT, Op0);
724 case MVT::v8bf16: return fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(RetVT, Op0);
725 case MVT::v2f32: return fastEmit_AArch64ISD_REV64_MVT_v2f32_r(RetVT, Op0);
726 case MVT::v4f32: return fastEmit_AArch64ISD_REV64_MVT_v4f32_r(RetVT, Op0);
727 default: return Register();
728 }
729}
730
731// FastEmit functions for AArch64ISD::SADDLP.
732
733Register fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(MVT RetVT, Register Op0) {
734 if (RetVT.SimpleTy != MVT::v4i16)
735 return Register();
736 if ((Subtarget->isNeonAvailable())) {
737 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv8i8_v4i16, RC: &AArch64::FPR64RegClass, Op0);
738 }
739 return Register();
740}
741
742Register fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(MVT RetVT, Register Op0) {
743 if (RetVT.SimpleTy != MVT::v8i16)
744 return Register();
745 if ((Subtarget->isNeonAvailable())) {
746 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv16i8_v8i16, RC: &AArch64::FPR128RegClass, Op0);
747 }
748 return Register();
749}
750
751Register fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(MVT RetVT, Register Op0) {
752 if (RetVT.SimpleTy != MVT::v2i32)
753 return Register();
754 if ((Subtarget->isNeonAvailable())) {
755 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv4i16_v2i32, RC: &AArch64::FPR64RegClass, Op0);
756 }
757 return Register();
758}
759
760Register fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(MVT RetVT, Register Op0) {
761 if (RetVT.SimpleTy != MVT::v4i32)
762 return Register();
763 if ((Subtarget->isNeonAvailable())) {
764 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv8i16_v4i32, RC: &AArch64::FPR128RegClass, Op0);
765 }
766 return Register();
767}
768
769Register fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(MVT RetVT, Register Op0) {
770 if (RetVT.SimpleTy != MVT::v1i64)
771 return Register();
772 if ((Subtarget->isNeonAvailable())) {
773 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv2i32_v1i64, RC: &AArch64::FPR64RegClass, Op0);
774 }
775 return Register();
776}
777
778Register fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(MVT RetVT, Register Op0) {
779 if (RetVT.SimpleTy != MVT::v2i64)
780 return Register();
781 if ((Subtarget->isNeonAvailable())) {
782 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv4i32_v2i64, RC: &AArch64::FPR128RegClass, Op0);
783 }
784 return Register();
785}
786
787Register fastEmit_AArch64ISD_SADDLP_r(MVT VT, MVT RetVT, Register Op0) {
788 switch (VT.SimpleTy) {
789 case MVT::v8i8: return fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(RetVT, Op0);
790 case MVT::v16i8: return fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(RetVT, Op0);
791 case MVT::v4i16: return fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(RetVT, Op0);
792 case MVT::v8i16: return fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(RetVT, Op0);
793 case MVT::v2i32: return fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(RetVT, Op0);
794 case MVT::v4i32: return fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(RetVT, Op0);
795 default: return Register();
796 }
797}
798
799// FastEmit functions for AArch64ISD::SITOF.
800
801Register fastEmit_AArch64ISD_SITOF_MVT_f16_r(MVT RetVT, Register Op0) {
802 if (RetVT.SimpleTy != MVT::f16)
803 return Register();
804 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
805 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i16, RC: &AArch64::FPR16RegClass, Op0);
806 }
807 return Register();
808}
809
810Register fastEmit_AArch64ISD_SITOF_MVT_f32_r(MVT RetVT, Register Op0) {
811 if (RetVT.SimpleTy != MVT::f32)
812 return Register();
813 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
814 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i32, RC: &AArch64::FPR32RegClass, Op0);
815 }
816 return Register();
817}
818
819Register fastEmit_AArch64ISD_SITOF_MVT_f64_r(MVT RetVT, Register Op0) {
820 if (RetVT.SimpleTy != MVT::f64)
821 return Register();
822 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
823 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i64, RC: &AArch64::FPR64RegClass, Op0);
824 }
825 return Register();
826}
827
828Register fastEmit_AArch64ISD_SITOF_r(MVT VT, MVT RetVT, Register Op0) {
829 switch (VT.SimpleTy) {
830 case MVT::f16: return fastEmit_AArch64ISD_SITOF_MVT_f16_r(RetVT, Op0);
831 case MVT::f32: return fastEmit_AArch64ISD_SITOF_MVT_f32_r(RetVT, Op0);
832 case MVT::f64: return fastEmit_AArch64ISD_SITOF_MVT_f64_r(RetVT, Op0);
833 default: return Register();
834 }
835}
836
837// FastEmit functions for AArch64ISD::SQABS.
838
839Register fastEmit_AArch64ISD_SQABS_MVT_f32_r(MVT RetVT, Register Op0) {
840 if (RetVT.SimpleTy != MVT::f32)
841 return Register();
842 if ((Subtarget->isNeonAvailable())) {
843 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv1i32, RC: &AArch64::FPR32RegClass, Op0);
844 }
845 return Register();
846}
847
848Register fastEmit_AArch64ISD_SQABS_MVT_f64_r(MVT RetVT, Register Op0) {
849 if (RetVT.SimpleTy != MVT::f64)
850 return Register();
851 if ((Subtarget->isNeonAvailable())) {
852 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
853 }
854 return Register();
855}
856
857Register fastEmit_AArch64ISD_SQABS_r(MVT VT, MVT RetVT, Register Op0) {
858 switch (VT.SimpleTy) {
859 case MVT::f32: return fastEmit_AArch64ISD_SQABS_MVT_f32_r(RetVT, Op0);
860 case MVT::f64: return fastEmit_AArch64ISD_SQABS_MVT_f64_r(RetVT, Op0);
861 default: return Register();
862 }
863}
864
865// FastEmit functions for AArch64ISD::SQNEG.
866
867Register fastEmit_AArch64ISD_SQNEG_MVT_f32_r(MVT RetVT, Register Op0) {
868 if (RetVT.SimpleTy != MVT::f32)
869 return Register();
870 if ((Subtarget->isNeonAvailable())) {
871 return fastEmitInst_r(MachineInstOpcode: AArch64::SQNEGv1i32, RC: &AArch64::FPR32RegClass, Op0);
872 }
873 return Register();
874}
875
876Register fastEmit_AArch64ISD_SQNEG_MVT_f64_r(MVT RetVT, Register Op0) {
877 if (RetVT.SimpleTy != MVT::f64)
878 return Register();
879 if ((Subtarget->isNeonAvailable())) {
880 return fastEmitInst_r(MachineInstOpcode: AArch64::SQNEGv1i64, RC: &AArch64::FPR64RegClass, Op0);
881 }
882 return Register();
883}
884
885Register fastEmit_AArch64ISD_SQNEG_r(MVT VT, MVT RetVT, Register Op0) {
886 switch (VT.SimpleTy) {
887 case MVT::f32: return fastEmit_AArch64ISD_SQNEG_MVT_f32_r(RetVT, Op0);
888 case MVT::f64: return fastEmit_AArch64ISD_SQNEG_MVT_f64_r(RetVT, Op0);
889 default: return Register();
890 }
891}
892
893// FastEmit functions for AArch64ISD::SUNPKHI.
894
895Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
896 if (RetVT.SimpleTy != MVT::nxv8i16)
897 return Register();
898 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
899 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
900 }
901 return Register();
902}
903
904Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
905 if (RetVT.SimpleTy != MVT::nxv4i32)
906 return Register();
907 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
908 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
909 }
910 return Register();
911}
912
913Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
914 if (RetVT.SimpleTy != MVT::nxv2i64)
915 return Register();
916 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
917 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
918 }
919 return Register();
920}
921
922Register fastEmit_AArch64ISD_SUNPKHI_r(MVT VT, MVT RetVT, Register Op0) {
923 switch (VT.SimpleTy) {
924 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(RetVT, Op0);
925 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(RetVT, Op0);
926 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(RetVT, Op0);
927 default: return Register();
928 }
929}
930
931// FastEmit functions for AArch64ISD::SUNPKLO.
932
933Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
934 if (RetVT.SimpleTy != MVT::nxv8i16)
935 return Register();
936 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
937 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
938 }
939 return Register();
940}
941
942Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
943 if (RetVT.SimpleTy != MVT::nxv4i32)
944 return Register();
945 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
946 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
947 }
948 return Register();
949}
950
951Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
952 if (RetVT.SimpleTy != MVT::nxv2i64)
953 return Register();
954 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
955 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
956 }
957 return Register();
958}
959
960Register fastEmit_AArch64ISD_SUNPKLO_r(MVT VT, MVT RetVT, Register Op0) {
961 switch (VT.SimpleTy) {
962 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(RetVT, Op0);
963 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(RetVT, Op0);
964 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(RetVT, Op0);
965 default: return Register();
966 }
967}
968
969// FastEmit functions for AArch64ISD::UADDLP.
970
971Register fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(MVT RetVT, Register Op0) {
972 if (RetVT.SimpleTy != MVT::v4i16)
973 return Register();
974 if ((Subtarget->isNeonAvailable())) {
975 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv8i8_v4i16, RC: &AArch64::FPR64RegClass, Op0);
976 }
977 return Register();
978}
979
980Register fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(MVT RetVT, Register Op0) {
981 if (RetVT.SimpleTy != MVT::v8i16)
982 return Register();
983 if ((Subtarget->isNeonAvailable())) {
984 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv16i8_v8i16, RC: &AArch64::FPR128RegClass, Op0);
985 }
986 return Register();
987}
988
989Register fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(MVT RetVT, Register Op0) {
990 if (RetVT.SimpleTy != MVT::v2i32)
991 return Register();
992 if ((Subtarget->isNeonAvailable())) {
993 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv4i16_v2i32, RC: &AArch64::FPR64RegClass, Op0);
994 }
995 return Register();
996}
997
998Register fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(MVT RetVT, Register Op0) {
999 if (RetVT.SimpleTy != MVT::v4i32)
1000 return Register();
1001 if ((Subtarget->isNeonAvailable())) {
1002 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv8i16_v4i32, RC: &AArch64::FPR128RegClass, Op0);
1003 }
1004 return Register();
1005}
1006
1007Register fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(MVT RetVT, Register Op0) {
1008 if (RetVT.SimpleTy != MVT::v1i64)
1009 return Register();
1010 if ((Subtarget->isNeonAvailable())) {
1011 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv2i32_v1i64, RC: &AArch64::FPR64RegClass, Op0);
1012 }
1013 return Register();
1014}
1015
1016Register fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(MVT RetVT, Register Op0) {
1017 if (RetVT.SimpleTy != MVT::v2i64)
1018 return Register();
1019 if ((Subtarget->isNeonAvailable())) {
1020 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv4i32_v2i64, RC: &AArch64::FPR128RegClass, Op0);
1021 }
1022 return Register();
1023}
1024
1025Register fastEmit_AArch64ISD_UADDLP_r(MVT VT, MVT RetVT, Register Op0) {
1026 switch (VT.SimpleTy) {
1027 case MVT::v8i8: return fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(RetVT, Op0);
1028 case MVT::v16i8: return fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(RetVT, Op0);
1029 case MVT::v4i16: return fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(RetVT, Op0);
1030 case MVT::v8i16: return fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(RetVT, Op0);
1031 case MVT::v2i32: return fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(RetVT, Op0);
1032 case MVT::v4i32: return fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(RetVT, Op0);
1033 default: return Register();
1034 }
1035}
1036
1037// FastEmit functions for AArch64ISD::UITOF.
1038
1039Register fastEmit_AArch64ISD_UITOF_MVT_f16_r(MVT RetVT, Register Op0) {
1040 if (RetVT.SimpleTy != MVT::f16)
1041 return Register();
1042 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
1043 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i16, RC: &AArch64::FPR16RegClass, Op0);
1044 }
1045 return Register();
1046}
1047
1048Register fastEmit_AArch64ISD_UITOF_MVT_f32_r(MVT RetVT, Register Op0) {
1049 if (RetVT.SimpleTy != MVT::f32)
1050 return Register();
1051 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
1052 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i32, RC: &AArch64::FPR32RegClass, Op0);
1053 }
1054 return Register();
1055}
1056
1057Register fastEmit_AArch64ISD_UITOF_MVT_f64_r(MVT RetVT, Register Op0) {
1058 if (RetVT.SimpleTy != MVT::f64)
1059 return Register();
1060 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
1061 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i64, RC: &AArch64::FPR64RegClass, Op0);
1062 }
1063 return Register();
1064}
1065
1066Register fastEmit_AArch64ISD_UITOF_r(MVT VT, MVT RetVT, Register Op0) {
1067 switch (VT.SimpleTy) {
1068 case MVT::f16: return fastEmit_AArch64ISD_UITOF_MVT_f16_r(RetVT, Op0);
1069 case MVT::f32: return fastEmit_AArch64ISD_UITOF_MVT_f32_r(RetVT, Op0);
1070 case MVT::f64: return fastEmit_AArch64ISD_UITOF_MVT_f64_r(RetVT, Op0);
1071 default: return Register();
1072 }
1073}
1074
1075// FastEmit functions for AArch64ISD::UUNPKHI.
1076
1077Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
1078 if (RetVT.SimpleTy != MVT::nxv8i16)
1079 return Register();
1080 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1081 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1082 }
1083 return Register();
1084}
1085
1086Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
1087 if (RetVT.SimpleTy != MVT::nxv4i32)
1088 return Register();
1089 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1090 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1091 }
1092 return Register();
1093}
1094
1095Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
1096 if (RetVT.SimpleTy != MVT::nxv2i64)
1097 return Register();
1098 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1099 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1100 }
1101 return Register();
1102}
1103
1104Register fastEmit_AArch64ISD_UUNPKHI_r(MVT VT, MVT RetVT, Register Op0) {
1105 switch (VT.SimpleTy) {
1106 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(RetVT, Op0);
1107 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(RetVT, Op0);
1108 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(RetVT, Op0);
1109 default: return Register();
1110 }
1111}
1112
1113// FastEmit functions for AArch64ISD::UUNPKLO.
1114
1115Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
1116 if (RetVT.SimpleTy != MVT::nxv8i16)
1117 return Register();
1118 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1119 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1120 }
1121 return Register();
1122}
1123
1124Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
1125 if (RetVT.SimpleTy != MVT::nxv4i32)
1126 return Register();
1127 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1128 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1129 }
1130 return Register();
1131}
1132
1133Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
1134 if (RetVT.SimpleTy != MVT::nxv2i64)
1135 return Register();
1136 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1137 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1138 }
1139 return Register();
1140}
1141
1142Register fastEmit_AArch64ISD_UUNPKLO_r(MVT VT, MVT RetVT, Register Op0) {
1143 switch (VT.SimpleTy) {
1144 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(RetVT, Op0);
1145 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(RetVT, Op0);
1146 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(RetVT, Op0);
1147 default: return Register();
1148 }
1149}
1150
1151// FastEmit functions for ISD::ABS.
1152
1153Register fastEmit_ISD_ABS_MVT_i32_r(MVT RetVT, Register Op0) {
1154 if (RetVT.SimpleTy != MVT::i32)
1155 return Register();
1156 if ((Subtarget->hasCSSC())) {
1157 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSWr, RC: &AArch64::GPR32RegClass, Op0);
1158 }
1159 return Register();
1160}
1161
1162Register fastEmit_ISD_ABS_MVT_i64_r(MVT RetVT, Register Op0) {
1163 if (RetVT.SimpleTy != MVT::i64)
1164 return Register();
1165 if ((!Subtarget->hasCSSC())) {
1166 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
1167 }
1168 if ((Subtarget->hasCSSC())) {
1169 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSXr, RC: &AArch64::GPR64RegClass, Op0);
1170 }
1171 return Register();
1172}
1173
1174Register fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, Register Op0) {
1175 if (RetVT.SimpleTy != MVT::v8i8)
1176 return Register();
1177 if ((Subtarget->isNeonAvailable())) {
1178 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv8i8, RC: &AArch64::FPR64RegClass, Op0);
1179 }
1180 return Register();
1181}
1182
1183Register fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, Register Op0) {
1184 if (RetVT.SimpleTy != MVT::v16i8)
1185 return Register();
1186 if ((Subtarget->isNeonAvailable())) {
1187 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv16i8, RC: &AArch64::FPR128RegClass, Op0);
1188 }
1189 return Register();
1190}
1191
1192Register fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, Register Op0) {
1193 if (RetVT.SimpleTy != MVT::v4i16)
1194 return Register();
1195 if ((Subtarget->isNeonAvailable())) {
1196 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv4i16, RC: &AArch64::FPR64RegClass, Op0);
1197 }
1198 return Register();
1199}
1200
1201Register fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, Register Op0) {
1202 if (RetVT.SimpleTy != MVT::v8i16)
1203 return Register();
1204 if ((Subtarget->isNeonAvailable())) {
1205 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv8i16, RC: &AArch64::FPR128RegClass, Op0);
1206 }
1207 return Register();
1208}
1209
1210Register fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, Register Op0) {
1211 if (RetVT.SimpleTy != MVT::v2i32)
1212 return Register();
1213 if ((Subtarget->isNeonAvailable())) {
1214 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv2i32, RC: &AArch64::FPR64RegClass, Op0);
1215 }
1216 return Register();
1217}
1218
1219Register fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, Register Op0) {
1220 if (RetVT.SimpleTy != MVT::v4i32)
1221 return Register();
1222 if ((Subtarget->isNeonAvailable())) {
1223 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv4i32, RC: &AArch64::FPR128RegClass, Op0);
1224 }
1225 return Register();
1226}
1227
1228Register fastEmit_ISD_ABS_MVT_v1i64_r(MVT RetVT, Register Op0) {
1229 if (RetVT.SimpleTy != MVT::v1i64)
1230 return Register();
1231 if ((Subtarget->isNeonAvailable())) {
1232 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
1233 }
1234 return Register();
1235}
1236
1237Register fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, Register Op0) {
1238 if (RetVT.SimpleTy != MVT::v2i64)
1239 return Register();
1240 if ((Subtarget->isNeonAvailable())) {
1241 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv2i64, RC: &AArch64::FPR128RegClass, Op0);
1242 }
1243 return Register();
1244}
1245
1246Register fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, Register Op0) {
1247 switch (VT.SimpleTy) {
1248 case MVT::i32: return fastEmit_ISD_ABS_MVT_i32_r(RetVT, Op0);
1249 case MVT::i64: return fastEmit_ISD_ABS_MVT_i64_r(RetVT, Op0);
1250 case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0);
1251 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
1252 case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0);
1253 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
1254 case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0);
1255 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
1256 case MVT::v1i64: return fastEmit_ISD_ABS_MVT_v1i64_r(RetVT, Op0);
1257 case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0);
1258 default: return Register();
1259 }
1260}
1261
1262// FastEmit functions for ISD::BITCAST.
1263
1264Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Register Op0) {
1265 if ((!Subtarget->isLittleEndian())) {
1266 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1267 }
1268 return Register();
1269}
1270
1271Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Register Op0) {
1272 if ((!Subtarget->isLittleEndian())) {
1273 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1274 }
1275 return Register();
1276}
1277
1278Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Register Op0) {
1279 if ((!Subtarget->isLittleEndian())) {
1280 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1281 }
1282 return Register();
1283}
1284
1285Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Register Op0) {
1286 if ((!Subtarget->isLittleEndian())) {
1287 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1288 }
1289 return Register();
1290}
1291
1292Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Register Op0) {
1293 if ((!Subtarget->isLittleEndian())) {
1294 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1295 }
1296 return Register();
1297}
1298
1299Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Register Op0) {
1300 if ((!Subtarget->isLittleEndian())) {
1301 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1302 }
1303 return Register();
1304}
1305
1306Register fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, Register Op0) {
1307switch (RetVT.SimpleTy) {
1308 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0);
1309 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0);
1310 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0);
1311 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0);
1312 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Op0);
1313 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0);
1314 default: return Register();
1315}
1316}
1317
1318Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Register Op0) {
1319 if ((!Subtarget->isLittleEndian())) {
1320 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1321 }
1322 return Register();
1323}
1324
1325Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Register Op0) {
1326 if ((!Subtarget->isLittleEndian())) {
1327 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1328 }
1329 return Register();
1330}
1331
1332Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Register Op0) {
1333 if ((!Subtarget->isLittleEndian())) {
1334 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1335 }
1336 return Register();
1337}
1338
1339Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Register Op0) {
1340 if ((!Subtarget->isLittleEndian())) {
1341 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1342 }
1343 return Register();
1344}
1345
1346Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Register Op0) {
1347 if ((!Subtarget->isLittleEndian())) {
1348 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1349 }
1350 return Register();
1351}
1352
1353Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Register Op0) {
1354 if ((!Subtarget->isLittleEndian())) {
1355 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1356 }
1357 return Register();
1358}
1359
1360Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Register Op0) {
1361 if ((!Subtarget->isLittleEndian())) {
1362 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1363 }
1364 return Register();
1365}
1366
1367Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Register Op0) {
1368 if ((!Subtarget->isLittleEndian())) {
1369 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1370 }
1371 return Register();
1372}
1373
1374Register fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, Register Op0) {
1375switch (RetVT.SimpleTy) {
1376 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0);
1377 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0);
1378 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0);
1379 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0);
1380 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0);
1381 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Op0);
1382 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0);
1383 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Op0);
1384 default: return Register();
1385}
1386}
1387
1388Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Register Op0) {
1389 if ((!Subtarget->isLittleEndian())) {
1390 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1391 }
1392 return Register();
1393}
1394
1395Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Register Op0) {
1396 if ((!Subtarget->isLittleEndian())) {
1397 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1398 }
1399 return Register();
1400}
1401
1402Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Register Op0) {
1403 if ((!Subtarget->isLittleEndian())) {
1404 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1405 }
1406 return Register();
1407}
1408
1409Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Register Op0) {
1410 if ((!Subtarget->isLittleEndian())) {
1411 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1412 }
1413 return Register();
1414}
1415
1416Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Register Op0) {
1417 if ((!Subtarget->isLittleEndian())) {
1418 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1419 }
1420 return Register();
1421}
1422
1423Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Register Op0) {
1424 if ((!Subtarget->isLittleEndian())) {
1425 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1426 }
1427 return Register();
1428}
1429
1430Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Register Op0) {
1431 if ((!Subtarget->isLittleEndian())) {
1432 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1433 }
1434 return Register();
1435}
1436
1437Register fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, Register Op0) {
1438switch (RetVT.SimpleTy) {
1439 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0);
1440 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0);
1441 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0);
1442 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0);
1443 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Op0);
1444 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0);
1445 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0);
1446 default: return Register();
1447}
1448}
1449
1450Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Register Op0) {
1451 if ((!Subtarget->isLittleEndian())) {
1452 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1453 }
1454 return Register();
1455}
1456
1457Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Register Op0) {
1458 if ((!Subtarget->isLittleEndian())) {
1459 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1460 }
1461 return Register();
1462}
1463
1464Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Register Op0) {
1465 if ((!Subtarget->isLittleEndian())) {
1466 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1467 }
1468 return Register();
1469}
1470
1471Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Register Op0) {
1472 if ((!Subtarget->isLittleEndian())) {
1473 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1474 }
1475 return Register();
1476}
1477
1478Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Register Op0) {
1479 if ((!Subtarget->isLittleEndian())) {
1480 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1481 }
1482 return Register();
1483}
1484
1485Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Register Op0) {
1486 if ((!Subtarget->isLittleEndian())) {
1487 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1488 }
1489 return Register();
1490}
1491
1492Register fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, Register Op0) {
1493switch (RetVT.SimpleTy) {
1494 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0);
1495 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0);
1496 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0);
1497 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0);
1498 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0);
1499 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Op0);
1500 default: return Register();
1501}
1502}
1503
1504Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Register Op0) {
1505 if ((!Subtarget->isLittleEndian())) {
1506 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1507 }
1508 return Register();
1509}
1510
1511Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Register Op0) {
1512 if ((!Subtarget->isLittleEndian())) {
1513 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1514 }
1515 return Register();
1516}
1517
1518Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Register Op0) {
1519 if ((!Subtarget->isLittleEndian())) {
1520 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1521 }
1522 return Register();
1523}
1524
1525Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Register Op0) {
1526 if ((!Subtarget->isLittleEndian())) {
1527 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1528 }
1529 return Register();
1530}
1531
1532Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Register Op0) {
1533 if ((!Subtarget->isLittleEndian())) {
1534 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1535 }
1536 return Register();
1537}
1538
1539Register fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, Register Op0) {
1540switch (RetVT.SimpleTy) {
1541 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0);
1542 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0);
1543 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0);
1544 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0);
1545 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0);
1546 default: return Register();
1547}
1548}
1549
1550Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Register Op0) {
1551 if ((!Subtarget->isLittleEndian())) {
1552 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1553 }
1554 return Register();
1555}
1556
1557Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Register Op0) {
1558 if ((!Subtarget->isLittleEndian())) {
1559 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1560 }
1561 return Register();
1562}
1563
1564Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Register Op0) {
1565 if ((!Subtarget->isLittleEndian())) {
1566 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1567 }
1568 return Register();
1569}
1570
1571Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Register Op0) {
1572 if ((!Subtarget->isLittleEndian())) {
1573 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1574 }
1575 return Register();
1576}
1577
1578Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Register Op0) {
1579 if ((!Subtarget->isLittleEndian())) {
1580 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1581 }
1582 return Register();
1583}
1584
1585Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Register Op0) {
1586 if ((!Subtarget->isLittleEndian())) {
1587 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1588 }
1589 return Register();
1590}
1591
1592Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Register Op0) {
1593 if ((!Subtarget->isLittleEndian())) {
1594 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1595 }
1596 return Register();
1597}
1598
1599Register fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, Register Op0) {
1600switch (RetVT.SimpleTy) {
1601 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0);
1602 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0);
1603 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0);
1604 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0);
1605 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0);
1606 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Op0);
1607 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Op0);
1608 default: return Register();
1609}
1610}
1611
1612Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Register Op0) {
1613 if ((!Subtarget->isLittleEndian())) {
1614 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1615 }
1616 return Register();
1617}
1618
1619Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Register Op0) {
1620 if ((!Subtarget->isLittleEndian())) {
1621 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1622 }
1623 return Register();
1624}
1625
1626Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Register Op0) {
1627 if ((!Subtarget->isLittleEndian())) {
1628 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1629 }
1630 return Register();
1631}
1632
1633Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Register Op0) {
1634 if ((!Subtarget->isLittleEndian())) {
1635 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1636 }
1637 return Register();
1638}
1639
1640Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Register Op0) {
1641 if ((!Subtarget->isLittleEndian())) {
1642 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1643 }
1644 return Register();
1645}
1646
1647Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Register Op0) {
1648 if ((!Subtarget->isLittleEndian())) {
1649 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1650 }
1651 return Register();
1652}
1653
1654Register fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, Register Op0) {
1655switch (RetVT.SimpleTy) {
1656 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0);
1657 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0);
1658 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0);
1659 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0);
1660 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Op0);
1661 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0);
1662 default: return Register();
1663}
1664}
1665
1666Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Register Op0) {
1667 if ((!Subtarget->isLittleEndian())) {
1668 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1669 }
1670 return Register();
1671}
1672
1673Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Register Op0) {
1674 if ((!Subtarget->isLittleEndian())) {
1675 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1676 }
1677 return Register();
1678}
1679
1680Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Register Op0) {
1681 if ((!Subtarget->isLittleEndian())) {
1682 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1683 }
1684 return Register();
1685}
1686
1687Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Register Op0) {
1688 if ((!Subtarget->isLittleEndian())) {
1689 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1690 }
1691 return Register();
1692}
1693
1694Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Register Op0) {
1695 if ((!Subtarget->isLittleEndian())) {
1696 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1697 }
1698 return Register();
1699}
1700
1701Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Register Op0) {
1702 if ((!Subtarget->isLittleEndian())) {
1703 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1704 }
1705 return Register();
1706}
1707
1708Register fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, Register Op0) {
1709switch (RetVT.SimpleTy) {
1710 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0);
1711 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0);
1712 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0);
1713 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0);
1714 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Op0);
1715 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0);
1716 default: return Register();
1717}
1718}
1719
1720Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Register Op0) {
1721 if ((!Subtarget->isLittleEndian())) {
1722 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1723 }
1724 return Register();
1725}
1726
1727Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Register Op0) {
1728 if ((!Subtarget->isLittleEndian())) {
1729 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1730 }
1731 return Register();
1732}
1733
1734Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Register Op0) {
1735 if ((!Subtarget->isLittleEndian())) {
1736 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1737 }
1738 return Register();
1739}
1740
1741Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Register Op0) {
1742 if ((!Subtarget->isLittleEndian())) {
1743 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1744 }
1745 return Register();
1746}
1747
1748Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Register Op0) {
1749 if ((!Subtarget->isLittleEndian())) {
1750 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1751 }
1752 return Register();
1753}
1754
1755Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Register Op0) {
1756 if ((!Subtarget->isLittleEndian())) {
1757 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1758 }
1759 return Register();
1760}
1761
1762Register fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, Register Op0) {
1763switch (RetVT.SimpleTy) {
1764 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0);
1765 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0);
1766 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0);
1767 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0);
1768 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Op0);
1769 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0);
1770 default: return Register();
1771}
1772}
1773
1774Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Register Op0) {
1775 if ((!Subtarget->isLittleEndian())) {
1776 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1777 }
1778 return Register();
1779}
1780
1781Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Register Op0) {
1782 if ((!Subtarget->isLittleEndian())) {
1783 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1784 }
1785 return Register();
1786}
1787
1788Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Register Op0) {
1789 if ((!Subtarget->isLittleEndian())) {
1790 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1791 }
1792 return Register();
1793}
1794
1795Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Register Op0) {
1796 if ((!Subtarget->isLittleEndian())) {
1797 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1798 }
1799 return Register();
1800}
1801
1802Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Register Op0) {
1803 if ((!Subtarget->isLittleEndian())) {
1804 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1805 }
1806 return Register();
1807}
1808
1809Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Register Op0) {
1810 if ((!Subtarget->isLittleEndian())) {
1811 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1812 }
1813 return Register();
1814}
1815
1816Register fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, Register Op0) {
1817switch (RetVT.SimpleTy) {
1818 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0);
1819 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0);
1820 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0);
1821 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0);
1822 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0);
1823 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Op0);
1824 default: return Register();
1825}
1826}
1827
1828Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Register Op0) {
1829 if ((!Subtarget->isLittleEndian())) {
1830 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1831 }
1832 return Register();
1833}
1834
1835Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Register Op0) {
1836 if ((!Subtarget->isLittleEndian())) {
1837 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1838 }
1839 return Register();
1840}
1841
1842Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Register Op0) {
1843 if ((!Subtarget->isLittleEndian())) {
1844 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1845 }
1846 return Register();
1847}
1848
1849Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Register Op0) {
1850 if ((!Subtarget->isLittleEndian())) {
1851 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1852 }
1853 return Register();
1854}
1855
1856Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Register Op0) {
1857 if ((!Subtarget->isLittleEndian())) {
1858 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1859 }
1860 return Register();
1861}
1862
1863Register fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, Register Op0) {
1864switch (RetVT.SimpleTy) {
1865 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0);
1866 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0);
1867 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0);
1868 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0);
1869 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0);
1870 default: return Register();
1871}
1872}
1873
1874Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Register Op0) {
1875 if ((!Subtarget->isLittleEndian())) {
1876 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1877 }
1878 return Register();
1879}
1880
1881Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Register Op0) {
1882 if ((!Subtarget->isLittleEndian())) {
1883 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1884 }
1885 return Register();
1886}
1887
1888Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Register Op0) {
1889 if ((!Subtarget->isLittleEndian())) {
1890 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1891 }
1892 return Register();
1893}
1894
1895Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Register Op0) {
1896 if ((!Subtarget->isLittleEndian())) {
1897 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1898 }
1899 return Register();
1900}
1901
1902Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Register Op0) {
1903 if ((!Subtarget->isLittleEndian())) {
1904 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1905 }
1906 return Register();
1907}
1908
1909Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(Register Op0) {
1910 if ((!Subtarget->isLittleEndian())) {
1911 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1912 }
1913 return Register();
1914}
1915
1916Register fastEmit_ISD_BITCAST_MVT_v4bf16_r(MVT RetVT, Register Op0) {
1917switch (RetVT.SimpleTy) {
1918 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Op0);
1919 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Op0);
1920 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Op0);
1921 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Op0);
1922 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Op0);
1923 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(Op0);
1924 default: return Register();
1925}
1926}
1927
1928Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Register Op0) {
1929 if ((!Subtarget->isLittleEndian())) {
1930 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1931 }
1932 return Register();
1933}
1934
1935Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Register Op0) {
1936 if ((!Subtarget->isLittleEndian())) {
1937 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1938 }
1939 return Register();
1940}
1941
1942Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Register Op0) {
1943 if ((!Subtarget->isLittleEndian())) {
1944 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1945 }
1946 return Register();
1947}
1948
1949Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Register Op0) {
1950 if ((!Subtarget->isLittleEndian())) {
1951 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1952 }
1953 return Register();
1954}
1955
1956Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Register Op0) {
1957 if ((!Subtarget->isLittleEndian())) {
1958 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1959 }
1960 return Register();
1961}
1962
1963Register fastEmit_ISD_BITCAST_MVT_v8bf16_r(MVT RetVT, Register Op0) {
1964switch (RetVT.SimpleTy) {
1965 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Op0);
1966 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Op0);
1967 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Op0);
1968 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Op0);
1969 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Op0);
1970 default: return Register();
1971}
1972}
1973
1974Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Register Op0) {
1975 if ((!Subtarget->isLittleEndian())) {
1976 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1977 }
1978 return Register();
1979}
1980
1981Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Register Op0) {
1982 if ((!Subtarget->isLittleEndian())) {
1983 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1984 }
1985 return Register();
1986}
1987
1988Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Register Op0) {
1989 if ((!Subtarget->isLittleEndian())) {
1990 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1991 }
1992 return Register();
1993}
1994
1995Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Register Op0) {
1996 if ((!Subtarget->isLittleEndian())) {
1997 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1998 }
1999 return Register();
2000}
2001
2002Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Register Op0) {
2003 if ((!Subtarget->isLittleEndian())) {
2004 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2005 }
2006 return Register();
2007}
2008
2009Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Register Op0) {
2010 if ((!Subtarget->isLittleEndian())) {
2011 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2012 }
2013 return Register();
2014}
2015
2016Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Register Op0) {
2017 if ((!Subtarget->isLittleEndian())) {
2018 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2019 }
2020 return Register();
2021}
2022
2023Register fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, Register Op0) {
2024switch (RetVT.SimpleTy) {
2025 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0);
2026 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0);
2027 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0);
2028 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0);
2029 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0);
2030 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Op0);
2031 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Op0);
2032 default: return Register();
2033}
2034}
2035
2036Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Register Op0) {
2037 if ((!Subtarget->isLittleEndian())) {
2038 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
2039 }
2040 return Register();
2041}
2042
2043Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Register Op0) {
2044 if ((!Subtarget->isLittleEndian())) {
2045 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2046 }
2047 return Register();
2048}
2049
2050Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Register Op0) {
2051 if ((!Subtarget->isLittleEndian())) {
2052 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2053 }
2054 return Register();
2055}
2056
2057Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Register Op0) {
2058 if ((!Subtarget->isLittleEndian())) {
2059 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2060 }
2061 return Register();
2062}
2063
2064Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Register Op0) {
2065 if ((!Subtarget->isLittleEndian())) {
2066 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2067 }
2068 return Register();
2069}
2070
2071Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Register Op0) {
2072 if ((!Subtarget->isLittleEndian())) {
2073 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2074 }
2075 return Register();
2076}
2077
2078Register fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, Register Op0) {
2079switch (RetVT.SimpleTy) {
2080 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0);
2081 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0);
2082 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0);
2083 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0);
2084 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Op0);
2085 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0);
2086 default: return Register();
2087}
2088}
2089
2090Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Register Op0) {
2091 if ((!Subtarget->isLittleEndian())) {
2092 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
2093 }
2094 return Register();
2095}
2096
2097Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Register Op0) {
2098 if ((!Subtarget->isLittleEndian())) {
2099 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2100 }
2101 return Register();
2102}
2103
2104Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Register Op0) {
2105 if ((!Subtarget->isLittleEndian())) {
2106 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2107 }
2108 return Register();
2109}
2110
2111Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Register Op0) {
2112 if ((!Subtarget->isLittleEndian())) {
2113 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2114 }
2115 return Register();
2116}
2117
2118Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(Register Op0) {
2119 if ((!Subtarget->isLittleEndian())) {
2120 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2121 }
2122 return Register();
2123}
2124
2125Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Register Op0) {
2126 if ((!Subtarget->isLittleEndian())) {
2127 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2128 }
2129 return Register();
2130}
2131
2132Register fastEmit_ISD_BITCAST_MVT_v1f64_r(MVT RetVT, Register Op0) {
2133switch (RetVT.SimpleTy) {
2134 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Op0);
2135 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Op0);
2136 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Op0);
2137 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Op0);
2138 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(Op0);
2139 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Op0);
2140 default: return Register();
2141}
2142}
2143
2144Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Register Op0) {
2145 if ((!Subtarget->isLittleEndian())) {
2146 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2147 }
2148 return Register();
2149}
2150
2151Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Register Op0) {
2152 if ((!Subtarget->isLittleEndian())) {
2153 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2154 }
2155 return Register();
2156}
2157
2158Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Register Op0) {
2159 if ((!Subtarget->isLittleEndian())) {
2160 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2161 }
2162 return Register();
2163}
2164
2165Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Register Op0) {
2166 if ((!Subtarget->isLittleEndian())) {
2167 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2168 }
2169 return Register();
2170}
2171
2172Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Register Op0) {
2173 if ((!Subtarget->isLittleEndian())) {
2174 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2175 }
2176 return Register();
2177}
2178
2179Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Register Op0) {
2180 if ((!Subtarget->isLittleEndian())) {
2181 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2182 }
2183 return Register();
2184}
2185
2186Register fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, Register Op0) {
2187switch (RetVT.SimpleTy) {
2188 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0);
2189 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0);
2190 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0);
2191 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0);
2192 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Op0);
2193 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0);
2194 default: return Register();
2195}
2196}
2197
2198Register fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, Register Op0) {
2199 switch (VT.SimpleTy) {
2200 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
2201 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0);
2202 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0);
2203 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0);
2204 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0);
2205 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0);
2206 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0);
2207 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0);
2208 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0);
2209 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0);
2210 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0);
2211 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v4bf16_r(RetVT, Op0);
2212 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v8bf16_r(RetVT, Op0);
2213 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0);
2214 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0);
2215 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v1f64_r(RetVT, Op0);
2216 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0);
2217 default: return Register();
2218 }
2219}
2220
2221// FastEmit functions for ISD::BITREVERSE.
2222
2223Register fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, Register Op0) {
2224 if (RetVT.SimpleTy != MVT::i32)
2225 return Register();
2226 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITWr, RC: &AArch64::GPR32RegClass, Op0);
2227}
2228
2229Register fastEmit_ISD_BITREVERSE_MVT_i64_r(MVT RetVT, Register Op0) {
2230 if (RetVT.SimpleTy != MVT::i64)
2231 return Register();
2232 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITXr, RC: &AArch64::GPR64RegClass, Op0);
2233}
2234
2235Register fastEmit_ISD_BITREVERSE_MVT_v8i8_r(MVT RetVT, Register Op0) {
2236 if (RetVT.SimpleTy != MVT::v8i8)
2237 return Register();
2238 if ((Subtarget->isNeonAvailable())) {
2239 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITv8i8, RC: &AArch64::FPR64RegClass, Op0);
2240 }
2241 return Register();
2242}
2243
2244Register fastEmit_ISD_BITREVERSE_MVT_v16i8_r(MVT RetVT, Register Op0) {
2245 if (RetVT.SimpleTy != MVT::v16i8)
2246 return Register();
2247 if ((Subtarget->isNeonAvailable())) {
2248 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITv16i8, RC: &AArch64::FPR128RegClass, Op0);
2249 }
2250 return Register();
2251}
2252
2253Register fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, Register Op0) {
2254 switch (VT.SimpleTy) {
2255 case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0);
2256 case MVT::i64: return fastEmit_ISD_BITREVERSE_MVT_i64_r(RetVT, Op0);
2257 case MVT::v8i8: return fastEmit_ISD_BITREVERSE_MVT_v8i8_r(RetVT, Op0);
2258 case MVT::v16i8: return fastEmit_ISD_BITREVERSE_MVT_v16i8_r(RetVT, Op0);
2259 default: return Register();
2260 }
2261}
2262
2263// FastEmit functions for ISD::BRIND.
2264
2265Register fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, Register Op0) {
2266 if (RetVT.SimpleTy != MVT::isVoid)
2267 return Register();
2268 return fastEmitInst_r(MachineInstOpcode: AArch64::BR, RC: &AArch64::GPR64RegClass, Op0);
2269}
2270
2271Register fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, Register Op0) {
2272 switch (VT.SimpleTy) {
2273 case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0);
2274 default: return Register();
2275 }
2276}
2277
2278// FastEmit functions for ISD::BSWAP.
2279
2280Register fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, Register Op0) {
2281 if (RetVT.SimpleTy != MVT::i32)
2282 return Register();
2283 return fastEmitInst_r(MachineInstOpcode: AArch64::REVWr, RC: &AArch64::GPR32RegClass, Op0);
2284}
2285
2286Register fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, Register Op0) {
2287 if (RetVT.SimpleTy != MVT::i64)
2288 return Register();
2289 return fastEmitInst_r(MachineInstOpcode: AArch64::REVXr, RC: &AArch64::GPR64RegClass, Op0);
2290}
2291
2292Register fastEmit_ISD_BSWAP_MVT_v4i16_r(MVT RetVT, Register Op0) {
2293 if (RetVT.SimpleTy != MVT::v4i16)
2294 return Register();
2295 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
2296}
2297
2298Register fastEmit_ISD_BSWAP_MVT_v8i16_r(MVT RetVT, Register Op0) {
2299 if (RetVT.SimpleTy != MVT::v8i16)
2300 return Register();
2301 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
2302}
2303
2304Register fastEmit_ISD_BSWAP_MVT_v2i32_r(MVT RetVT, Register Op0) {
2305 if (RetVT.SimpleTy != MVT::v2i32)
2306 return Register();
2307 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
2308}
2309
2310Register fastEmit_ISD_BSWAP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2311 if (RetVT.SimpleTy != MVT::v4i32)
2312 return Register();
2313 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
2314}
2315
2316Register fastEmit_ISD_BSWAP_MVT_v2i64_r(MVT RetVT, Register Op0) {
2317 if (RetVT.SimpleTy != MVT::v2i64)
2318 return Register();
2319 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2320}
2321
2322Register fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, Register Op0) {
2323 switch (VT.SimpleTy) {
2324 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
2325 case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0);
2326 case MVT::v4i16: return fastEmit_ISD_BSWAP_MVT_v4i16_r(RetVT, Op0);
2327 case MVT::v8i16: return fastEmit_ISD_BSWAP_MVT_v8i16_r(RetVT, Op0);
2328 case MVT::v2i32: return fastEmit_ISD_BSWAP_MVT_v2i32_r(RetVT, Op0);
2329 case MVT::v4i32: return fastEmit_ISD_BSWAP_MVT_v4i32_r(RetVT, Op0);
2330 case MVT::v2i64: return fastEmit_ISD_BSWAP_MVT_v2i64_r(RetVT, Op0);
2331 default: return Register();
2332 }
2333}
2334
2335// FastEmit functions for ISD::CTLS.
2336
2337Register fastEmit_ISD_CTLS_MVT_i32_r(MVT RetVT, Register Op0) {
2338 if (RetVT.SimpleTy != MVT::i32)
2339 return Register();
2340 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSWr, RC: &AArch64::GPR32RegClass, Op0);
2341}
2342
2343Register fastEmit_ISD_CTLS_MVT_i64_r(MVT RetVT, Register Op0) {
2344 if (RetVT.SimpleTy != MVT::i64)
2345 return Register();
2346 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSXr, RC: &AArch64::GPR64RegClass, Op0);
2347}
2348
2349Register fastEmit_ISD_CTLS_MVT_v8i8_r(MVT RetVT, Register Op0) {
2350 if (RetVT.SimpleTy != MVT::v8i8)
2351 return Register();
2352 if ((Subtarget->isNeonAvailable())) {
2353 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv8i8, RC: &AArch64::FPR64RegClass, Op0);
2354 }
2355 return Register();
2356}
2357
2358Register fastEmit_ISD_CTLS_MVT_v16i8_r(MVT RetVT, Register Op0) {
2359 if (RetVT.SimpleTy != MVT::v16i8)
2360 return Register();
2361 if ((Subtarget->isNeonAvailable())) {
2362 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv16i8, RC: &AArch64::FPR128RegClass, Op0);
2363 }
2364 return Register();
2365}
2366
2367Register fastEmit_ISD_CTLS_MVT_v4i16_r(MVT RetVT, Register Op0) {
2368 if (RetVT.SimpleTy != MVT::v4i16)
2369 return Register();
2370 if ((Subtarget->isNeonAvailable())) {
2371 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv4i16, RC: &AArch64::FPR64RegClass, Op0);
2372 }
2373 return Register();
2374}
2375
2376Register fastEmit_ISD_CTLS_MVT_v8i16_r(MVT RetVT, Register Op0) {
2377 if (RetVT.SimpleTy != MVT::v8i16)
2378 return Register();
2379 if ((Subtarget->isNeonAvailable())) {
2380 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv8i16, RC: &AArch64::FPR128RegClass, Op0);
2381 }
2382 return Register();
2383}
2384
2385Register fastEmit_ISD_CTLS_MVT_v2i32_r(MVT RetVT, Register Op0) {
2386 if (RetVT.SimpleTy != MVT::v2i32)
2387 return Register();
2388 if ((Subtarget->isNeonAvailable())) {
2389 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv2i32, RC: &AArch64::FPR64RegClass, Op0);
2390 }
2391 return Register();
2392}
2393
2394Register fastEmit_ISD_CTLS_MVT_v4i32_r(MVT RetVT, Register Op0) {
2395 if (RetVT.SimpleTy != MVT::v4i32)
2396 return Register();
2397 if ((Subtarget->isNeonAvailable())) {
2398 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv4i32, RC: &AArch64::FPR128RegClass, Op0);
2399 }
2400 return Register();
2401}
2402
2403Register fastEmit_ISD_CTLS_r(MVT VT, MVT RetVT, Register Op0) {
2404 switch (VT.SimpleTy) {
2405 case MVT::i32: return fastEmit_ISD_CTLS_MVT_i32_r(RetVT, Op0);
2406 case MVT::i64: return fastEmit_ISD_CTLS_MVT_i64_r(RetVT, Op0);
2407 case MVT::v8i8: return fastEmit_ISD_CTLS_MVT_v8i8_r(RetVT, Op0);
2408 case MVT::v16i8: return fastEmit_ISD_CTLS_MVT_v16i8_r(RetVT, Op0);
2409 case MVT::v4i16: return fastEmit_ISD_CTLS_MVT_v4i16_r(RetVT, Op0);
2410 case MVT::v8i16: return fastEmit_ISD_CTLS_MVT_v8i16_r(RetVT, Op0);
2411 case MVT::v2i32: return fastEmit_ISD_CTLS_MVT_v2i32_r(RetVT, Op0);
2412 case MVT::v4i32: return fastEmit_ISD_CTLS_MVT_v4i32_r(RetVT, Op0);
2413 default: return Register();
2414 }
2415}
2416
2417// FastEmit functions for ISD::CTLZ.
2418
2419Register fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, Register Op0) {
2420 if (RetVT.SimpleTy != MVT::i32)
2421 return Register();
2422 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZWr, RC: &AArch64::GPR32RegClass, Op0);
2423}
2424
2425Register fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, Register Op0) {
2426 if (RetVT.SimpleTy != MVT::i64)
2427 return Register();
2428 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZXr, RC: &AArch64::GPR64RegClass, Op0);
2429}
2430
2431Register fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, Register Op0) {
2432 if (RetVT.SimpleTy != MVT::v8i8)
2433 return Register();
2434 if ((Subtarget->isNeonAvailable())) {
2435 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv8i8, RC: &AArch64::FPR64RegClass, Op0);
2436 }
2437 return Register();
2438}
2439
2440Register fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, Register Op0) {
2441 if (RetVT.SimpleTy != MVT::v16i8)
2442 return Register();
2443 if ((Subtarget->isNeonAvailable())) {
2444 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv16i8, RC: &AArch64::FPR128RegClass, Op0);
2445 }
2446 return Register();
2447}
2448
2449Register fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, Register Op0) {
2450 if (RetVT.SimpleTy != MVT::v4i16)
2451 return Register();
2452 if ((Subtarget->isNeonAvailable())) {
2453 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv4i16, RC: &AArch64::FPR64RegClass, Op0);
2454 }
2455 return Register();
2456}
2457
2458Register fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, Register Op0) {
2459 if (RetVT.SimpleTy != MVT::v8i16)
2460 return Register();
2461 if ((Subtarget->isNeonAvailable())) {
2462 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv8i16, RC: &AArch64::FPR128RegClass, Op0);
2463 }
2464 return Register();
2465}
2466
2467Register fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, Register Op0) {
2468 if (RetVT.SimpleTy != MVT::v2i32)
2469 return Register();
2470 if ((Subtarget->isNeonAvailable())) {
2471 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv2i32, RC: &AArch64::FPR64RegClass, Op0);
2472 }
2473 return Register();
2474}
2475
2476Register fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, Register Op0) {
2477 if (RetVT.SimpleTy != MVT::v4i32)
2478 return Register();
2479 if ((Subtarget->isNeonAvailable())) {
2480 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv4i32, RC: &AArch64::FPR128RegClass, Op0);
2481 }
2482 return Register();
2483}
2484
2485Register fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, Register Op0) {
2486 switch (VT.SimpleTy) {
2487 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
2488 case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0);
2489 case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0);
2490 case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0);
2491 case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0);
2492 case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0);
2493 case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0);
2494 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
2495 default: return Register();
2496 }
2497}
2498
2499// FastEmit functions for ISD::CTPOP.
2500
2501Register fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, Register Op0) {
2502 if (RetVT.SimpleTy != MVT::i32)
2503 return Register();
2504 if ((Subtarget->hasCSSC())) {
2505 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTWr, RC: &AArch64::GPR32RegClass, Op0);
2506 }
2507 return Register();
2508}
2509
2510Register fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, Register Op0) {
2511 if (RetVT.SimpleTy != MVT::i64)
2512 return Register();
2513 if ((Subtarget->hasCSSC())) {
2514 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTXr, RC: &AArch64::GPR64RegClass, Op0);
2515 }
2516 return Register();
2517}
2518
2519Register fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, Register Op0) {
2520 if (RetVT.SimpleTy != MVT::v8i8)
2521 return Register();
2522 if ((Subtarget->isNeonAvailable())) {
2523 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTv8i8, RC: &AArch64::FPR64RegClass, Op0);
2524 }
2525 return Register();
2526}
2527
2528Register fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, Register Op0) {
2529 if (RetVT.SimpleTy != MVT::v16i8)
2530 return Register();
2531 if ((Subtarget->isNeonAvailable())) {
2532 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTv16i8, RC: &AArch64::FPR128RegClass, Op0);
2533 }
2534 return Register();
2535}
2536
2537Register fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, Register Op0) {
2538 switch (VT.SimpleTy) {
2539 case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0);
2540 case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0);
2541 case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0);
2542 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
2543 default: return Register();
2544 }
2545}
2546
2547// FastEmit functions for ISD::CTTZ.
2548
2549Register fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, Register Op0) {
2550 if (RetVT.SimpleTy != MVT::i32)
2551 return Register();
2552 if ((Subtarget->hasCSSC())) {
2553 return fastEmitInst_r(MachineInstOpcode: AArch64::CTZWr, RC: &AArch64::GPR32RegClass, Op0);
2554 }
2555 return Register();
2556}
2557
2558Register fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, Register Op0) {
2559 if (RetVT.SimpleTy != MVT::i64)
2560 return Register();
2561 if ((Subtarget->hasCSSC())) {
2562 return fastEmitInst_r(MachineInstOpcode: AArch64::CTZXr, RC: &AArch64::GPR64RegClass, Op0);
2563 }
2564 return Register();
2565}
2566
2567Register fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, Register Op0) {
2568 switch (VT.SimpleTy) {
2569 case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0);
2570 case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0);
2571 default: return Register();
2572 }
2573}
2574
2575// FastEmit functions for ISD::FABS.
2576
2577Register fastEmit_ISD_FABS_MVT_bf16_r(MVT RetVT, Register Op0) {
2578 if (RetVT.SimpleTy != MVT::bf16)
2579 return Register();
2580 if ((Subtarget->hasFullFP16())) {
2581 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSHr, RC: &AArch64::FPR16RegClass, Op0);
2582 }
2583 return Register();
2584}
2585
2586Register fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, Register Op0) {
2587 if (RetVT.SimpleTy != MVT::f16)
2588 return Register();
2589 if ((Subtarget->hasFullFP16())) {
2590 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSHr, RC: &AArch64::FPR16RegClass, Op0);
2591 }
2592 return Register();
2593}
2594
2595Register fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, Register Op0) {
2596 if (RetVT.SimpleTy != MVT::f32)
2597 return Register();
2598 if ((Subtarget->hasFPARMv8())) {
2599 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSSr, RC: &AArch64::FPR32RegClass, Op0);
2600 }
2601 return Register();
2602}
2603
2604Register fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, Register Op0) {
2605 if (RetVT.SimpleTy != MVT::f64)
2606 return Register();
2607 if ((Subtarget->hasFPARMv8())) {
2608 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSDr, RC: &AArch64::FPR64RegClass, Op0);
2609 }
2610 return Register();
2611}
2612
2613Register fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, Register Op0) {
2614 if (RetVT.SimpleTy != MVT::v4f16)
2615 return Register();
2616 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2617 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f16, RC: &AArch64::FPR64RegClass, Op0);
2618 }
2619 return Register();
2620}
2621
2622Register fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, Register Op0) {
2623 if (RetVT.SimpleTy != MVT::v8f16)
2624 return Register();
2625 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2626 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv8f16, RC: &AArch64::FPR128RegClass, Op0);
2627 }
2628 return Register();
2629}
2630
2631Register fastEmit_ISD_FABS_MVT_v4bf16_r(MVT RetVT, Register Op0) {
2632 if (RetVT.SimpleTy != MVT::v4bf16)
2633 return Register();
2634 if ((Subtarget->hasFullFP16())) {
2635 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f16, RC: &AArch64::FPR64RegClass, Op0);
2636 }
2637 return Register();
2638}
2639
2640Register fastEmit_ISD_FABS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
2641 if (RetVT.SimpleTy != MVT::v8bf16)
2642 return Register();
2643 if ((Subtarget->hasFullFP16())) {
2644 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv8f16, RC: &AArch64::FPR128RegClass, Op0);
2645 }
2646 return Register();
2647}
2648
2649Register fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, Register Op0) {
2650 if (RetVT.SimpleTy != MVT::v2f32)
2651 return Register();
2652 if ((Subtarget->isNeonAvailable())) {
2653 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv2f32, RC: &AArch64::FPR64RegClass, Op0);
2654 }
2655 return Register();
2656}
2657
2658Register fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, Register Op0) {
2659 if (RetVT.SimpleTy != MVT::v4f32)
2660 return Register();
2661 if ((Subtarget->isNeonAvailable())) {
2662 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f32, RC: &AArch64::FPR128RegClass, Op0);
2663 }
2664 return Register();
2665}
2666
2667Register fastEmit_ISD_FABS_MVT_v2f64_r(MVT RetVT, Register Op0) {
2668 if (RetVT.SimpleTy != MVT::v2f64)
2669 return Register();
2670 if ((Subtarget->isNeonAvailable())) {
2671 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv2f64, RC: &AArch64::FPR128RegClass, Op0);
2672 }
2673 return Register();
2674}
2675
2676Register fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, Register Op0) {
2677 switch (VT.SimpleTy) {
2678 case MVT::bf16: return fastEmit_ISD_FABS_MVT_bf16_r(RetVT, Op0);
2679 case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0);
2680 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
2681 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
2682 case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0);
2683 case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0);
2684 case MVT::v4bf16: return fastEmit_ISD_FABS_MVT_v4bf16_r(RetVT, Op0);
2685 case MVT::v8bf16: return fastEmit_ISD_FABS_MVT_v8bf16_r(RetVT, Op0);
2686 case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0);
2687 case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0);
2688 case MVT::v2f64: return fastEmit_ISD_FABS_MVT_v2f64_r(RetVT, Op0);
2689 default: return Register();
2690 }
2691}
2692
2693// FastEmit functions for ISD::FCEIL.
2694
2695Register fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
2696 if (RetVT.SimpleTy != MVT::f16)
2697 return Register();
2698 if ((Subtarget->hasFullFP16())) {
2699 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPHr, RC: &AArch64::FPR16RegClass, Op0);
2700 }
2701 return Register();
2702}
2703
2704Register fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
2705 if (RetVT.SimpleTy != MVT::f32)
2706 return Register();
2707 if ((Subtarget->hasFPARMv8())) {
2708 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPSr, RC: &AArch64::FPR32RegClass, Op0);
2709 }
2710 return Register();
2711}
2712
2713Register fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
2714 if (RetVT.SimpleTy != MVT::f64)
2715 return Register();
2716 if ((Subtarget->hasFPARMv8())) {
2717 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPDr, RC: &AArch64::FPR64RegClass, Op0);
2718 }
2719 return Register();
2720}
2721
2722Register fastEmit_ISD_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
2723 if (RetVT.SimpleTy != MVT::v4f16)
2724 return Register();
2725 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2726 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f16, RC: &AArch64::FPR64RegClass, Op0);
2727 }
2728 return Register();
2729}
2730
2731Register fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
2732 if (RetVT.SimpleTy != MVT::v8f16)
2733 return Register();
2734 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2735 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv8f16, RC: &AArch64::FPR128RegClass, Op0);
2736 }
2737 return Register();
2738}
2739
2740Register fastEmit_ISD_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
2741 if (RetVT.SimpleTy != MVT::v2f32)
2742 return Register();
2743 if ((Subtarget->isNeonAvailable())) {
2744 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f32, RC: &AArch64::FPR64RegClass, Op0);
2745 }
2746 return Register();
2747}
2748
2749Register fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
2750 if (RetVT.SimpleTy != MVT::v4f32)
2751 return Register();
2752 if ((Subtarget->isNeonAvailable())) {
2753 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f32, RC: &AArch64::FPR128RegClass, Op0);
2754 }
2755 return Register();
2756}
2757
2758Register fastEmit_ISD_FCEIL_MVT_v2f64_r(MVT RetVT, Register Op0) {
2759 if (RetVT.SimpleTy != MVT::v2f64)
2760 return Register();
2761 if ((Subtarget->isNeonAvailable())) {
2762 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f64, RC: &AArch64::FPR128RegClass, Op0);
2763 }
2764 return Register();
2765}
2766
2767Register fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
2768 switch (VT.SimpleTy) {
2769 case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0);
2770 case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0);
2771 case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0);
2772 case MVT::v4f16: return fastEmit_ISD_FCEIL_MVT_v4f16_r(RetVT, Op0);
2773 case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0);
2774 case MVT::v2f32: return fastEmit_ISD_FCEIL_MVT_v2f32_r(RetVT, Op0);
2775 case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0);
2776 case MVT::v2f64: return fastEmit_ISD_FCEIL_MVT_v2f64_r(RetVT, Op0);
2777 default: return Register();
2778 }
2779}
2780
2781// FastEmit functions for ISD::FFLOOR.
2782
2783Register fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
2784 if (RetVT.SimpleTy != MVT::f16)
2785 return Register();
2786 if ((Subtarget->hasFullFP16())) {
2787 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMHr, RC: &AArch64::FPR16RegClass, Op0);
2788 }
2789 return Register();
2790}
2791
2792Register fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
2793 if (RetVT.SimpleTy != MVT::f32)
2794 return Register();
2795 if ((Subtarget->hasFPARMv8())) {
2796 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMSr, RC: &AArch64::FPR32RegClass, Op0);
2797 }
2798 return Register();
2799}
2800
2801Register fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
2802 if (RetVT.SimpleTy != MVT::f64)
2803 return Register();
2804 if ((Subtarget->hasFPARMv8())) {
2805 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMDr, RC: &AArch64::FPR64RegClass, Op0);
2806 }
2807 return Register();
2808}
2809
2810Register fastEmit_ISD_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
2811 if (RetVT.SimpleTy != MVT::v4f16)
2812 return Register();
2813 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2814 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f16, RC: &AArch64::FPR64RegClass, Op0);
2815 }
2816 return Register();
2817}
2818
2819Register fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
2820 if (RetVT.SimpleTy != MVT::v8f16)
2821 return Register();
2822 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2823 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv8f16, RC: &AArch64::FPR128RegClass, Op0);
2824 }
2825 return Register();
2826}
2827
2828Register fastEmit_ISD_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
2829 if (RetVT.SimpleTy != MVT::v2f32)
2830 return Register();
2831 if ((Subtarget->isNeonAvailable())) {
2832 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f32, RC: &AArch64::FPR64RegClass, Op0);
2833 }
2834 return Register();
2835}
2836
2837Register fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
2838 if (RetVT.SimpleTy != MVT::v4f32)
2839 return Register();
2840 if ((Subtarget->isNeonAvailable())) {
2841 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f32, RC: &AArch64::FPR128RegClass, Op0);
2842 }
2843 return Register();
2844}
2845
2846Register fastEmit_ISD_FFLOOR_MVT_v2f64_r(MVT RetVT, Register Op0) {
2847 if (RetVT.SimpleTy != MVT::v2f64)
2848 return Register();
2849 if ((Subtarget->isNeonAvailable())) {
2850 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f64, RC: &AArch64::FPR128RegClass, Op0);
2851 }
2852 return Register();
2853}
2854
2855Register fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
2856 switch (VT.SimpleTy) {
2857 case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0);
2858 case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0);
2859 case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0);
2860 case MVT::v4f16: return fastEmit_ISD_FFLOOR_MVT_v4f16_r(RetVT, Op0);
2861 case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0);
2862 case MVT::v2f32: return fastEmit_ISD_FFLOOR_MVT_v2f32_r(RetVT, Op0);
2863 case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0);
2864 case MVT::v2f64: return fastEmit_ISD_FFLOOR_MVT_v2f64_r(RetVT, Op0);
2865 default: return Register();
2866 }
2867}
2868
2869// FastEmit functions for ISD::FNEARBYINT.
2870
2871Register fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
2872 if (RetVT.SimpleTy != MVT::f16)
2873 return Register();
2874 if ((Subtarget->hasFullFP16())) {
2875 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIHr, RC: &AArch64::FPR16RegClass, Op0);
2876 }
2877 return Register();
2878}
2879
2880Register fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
2881 if (RetVT.SimpleTy != MVT::f32)
2882 return Register();
2883 if ((Subtarget->hasFPARMv8())) {
2884 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTISr, RC: &AArch64::FPR32RegClass, Op0);
2885 }
2886 return Register();
2887}
2888
2889Register fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
2890 if (RetVT.SimpleTy != MVT::f64)
2891 return Register();
2892 if ((Subtarget->hasFPARMv8())) {
2893 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIDr, RC: &AArch64::FPR64RegClass, Op0);
2894 }
2895 return Register();
2896}
2897
2898Register fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2899 if (RetVT.SimpleTy != MVT::v4f16)
2900 return Register();
2901 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2902 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f16, RC: &AArch64::FPR64RegClass, Op0);
2903 }
2904 return Register();
2905}
2906
2907Register fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2908 if (RetVT.SimpleTy != MVT::v8f16)
2909 return Register();
2910 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2911 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv8f16, RC: &AArch64::FPR128RegClass, Op0);
2912 }
2913 return Register();
2914}
2915
2916Register fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2917 if (RetVT.SimpleTy != MVT::v2f32)
2918 return Register();
2919 if ((Subtarget->isNeonAvailable())) {
2920 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f32, RC: &AArch64::FPR64RegClass, Op0);
2921 }
2922 return Register();
2923}
2924
2925Register fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2926 if (RetVT.SimpleTy != MVT::v4f32)
2927 return Register();
2928 if ((Subtarget->isNeonAvailable())) {
2929 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f32, RC: &AArch64::FPR128RegClass, Op0);
2930 }
2931 return Register();
2932}
2933
2934Register fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
2935 if (RetVT.SimpleTy != MVT::v2f64)
2936 return Register();
2937 if ((Subtarget->isNeonAvailable())) {
2938 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f64, RC: &AArch64::FPR128RegClass, Op0);
2939 }
2940 return Register();
2941}
2942
2943Register fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
2944 switch (VT.SimpleTy) {
2945 case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0);
2946 case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0);
2947 case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0);
2948 case MVT::v4f16: return fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(RetVT, Op0);
2949 case MVT::v8f16: return fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(RetVT, Op0);
2950 case MVT::v2f32: return fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(RetVT, Op0);
2951 case MVT::v4f32: return fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(RetVT, Op0);
2952 case MVT::v2f64: return fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(RetVT, Op0);
2953 default: return Register();
2954 }
2955}
2956
2957// FastEmit functions for ISD::FNEG.
2958
2959Register fastEmit_ISD_FNEG_MVT_bf16_r(MVT RetVT, Register Op0) {
2960 if (RetVT.SimpleTy != MVT::bf16)
2961 return Register();
2962 if ((Subtarget->hasFullFP16())) {
2963 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGHr, RC: &AArch64::FPR16RegClass, Op0);
2964 }
2965 return Register();
2966}
2967
2968Register fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, Register Op0) {
2969 if (RetVT.SimpleTy != MVT::f16)
2970 return Register();
2971 if ((Subtarget->hasFullFP16())) {
2972 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGHr, RC: &AArch64::FPR16RegClass, Op0);
2973 }
2974 return Register();
2975}
2976
2977Register fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, Register Op0) {
2978 if (RetVT.SimpleTy != MVT::f32)
2979 return Register();
2980 if ((Subtarget->hasFPARMv8())) {
2981 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGSr, RC: &AArch64::FPR32RegClass, Op0);
2982 }
2983 return Register();
2984}
2985
2986Register fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, Register Op0) {
2987 if (RetVT.SimpleTy != MVT::f64)
2988 return Register();
2989 if ((Subtarget->hasFPARMv8())) {
2990 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGDr, RC: &AArch64::FPR64RegClass, Op0);
2991 }
2992 return Register();
2993}
2994
2995Register fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, Register Op0) {
2996 if (RetVT.SimpleTy != MVT::v4f16)
2997 return Register();
2998 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2999 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f16, RC: &AArch64::FPR64RegClass, Op0);
3000 }
3001 return Register();
3002}
3003
3004Register fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, Register Op0) {
3005 if (RetVT.SimpleTy != MVT::v8f16)
3006 return Register();
3007 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3008 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv8f16, RC: &AArch64::FPR128RegClass, Op0);
3009 }
3010 return Register();
3011}
3012
3013Register fastEmit_ISD_FNEG_MVT_v4bf16_r(MVT RetVT, Register Op0) {
3014 if (RetVT.SimpleTy != MVT::v4bf16)
3015 return Register();
3016 if ((Subtarget->hasFullFP16())) {
3017 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f16, RC: &AArch64::FPR64RegClass, Op0);
3018 }
3019 return Register();
3020}
3021
3022Register fastEmit_ISD_FNEG_MVT_v8bf16_r(MVT RetVT, Register Op0) {
3023 if (RetVT.SimpleTy != MVT::v8bf16)
3024 return Register();
3025 if ((Subtarget->hasFullFP16())) {
3026 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv8f16, RC: &AArch64::FPR128RegClass, Op0);
3027 }
3028 return Register();
3029}
3030
3031Register fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, Register Op0) {
3032 if (RetVT.SimpleTy != MVT::v2f32)
3033 return Register();
3034 if ((Subtarget->isNeonAvailable())) {
3035 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv2f32, RC: &AArch64::FPR64RegClass, Op0);
3036 }
3037 return Register();
3038}
3039
3040Register fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, Register Op0) {
3041 if (RetVT.SimpleTy != MVT::v4f32)
3042 return Register();
3043 if ((Subtarget->isNeonAvailable())) {
3044 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f32, RC: &AArch64::FPR128RegClass, Op0);
3045 }
3046 return Register();
3047}
3048
3049Register fastEmit_ISD_FNEG_MVT_v2f64_r(MVT RetVT, Register Op0) {
3050 if (RetVT.SimpleTy != MVT::v2f64)
3051 return Register();
3052 if ((Subtarget->isNeonAvailable())) {
3053 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv2f64, RC: &AArch64::FPR128RegClass, Op0);
3054 }
3055 return Register();
3056}
3057
3058Register fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, Register Op0) {
3059 switch (VT.SimpleTy) {
3060 case MVT::bf16: return fastEmit_ISD_FNEG_MVT_bf16_r(RetVT, Op0);
3061 case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0);
3062 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
3063 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
3064 case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0);
3065 case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0);
3066 case MVT::v4bf16: return fastEmit_ISD_FNEG_MVT_v4bf16_r(RetVT, Op0);
3067 case MVT::v8bf16: return fastEmit_ISD_FNEG_MVT_v8bf16_r(RetVT, Op0);
3068 case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0);
3069 case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0);
3070 case MVT::v2f64: return fastEmit_ISD_FNEG_MVT_v2f64_r(RetVT, Op0);
3071 default: return Register();
3072 }
3073}
3074
3075// FastEmit functions for ISD::FP_EXTEND.
3076
3077Register fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Register Op0) {
3078 if ((Subtarget->hasFPARMv8())) {
3079 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSHr, RC: &AArch64::FPR32RegClass, Op0);
3080 }
3081 return Register();
3082}
3083
3084Register fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Register Op0) {
3085 if ((Subtarget->hasFPARMv8())) {
3086 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDHr, RC: &AArch64::FPR64RegClass, Op0);
3087 }
3088 return Register();
3089}
3090
3091Register fastEmit_ISD_FP_EXTEND_MVT_f16_r(MVT RetVT, Register Op0) {
3092switch (RetVT.SimpleTy) {
3093 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Op0);
3094 case MVT::f64: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Op0);
3095 default: return Register();
3096}
3097}
3098
3099Register fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
3100 if (RetVT.SimpleTy != MVT::f64)
3101 return Register();
3102 if ((Subtarget->hasFPARMv8())) {
3103 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDSr, RC: &AArch64::FPR64RegClass, Op0);
3104 }
3105 return Register();
3106}
3107
3108Register fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3109 if (RetVT.SimpleTy != MVT::v4f32)
3110 return Register();
3111 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv4i16, RC: &AArch64::FPR128RegClass, Op0);
3112}
3113
3114Register fastEmit_ISD_FP_EXTEND_MVT_v4bf16_r(MVT RetVT, Register Op0) {
3115 if (RetVT.SimpleTy != MVT::v4f32)
3116 return Register();
3117 if ((Subtarget->isNeonAvailable())) {
3118 return fastEmitInst_r(MachineInstOpcode: AArch64::SHLLv4i16, RC: &AArch64::FPR128RegClass, Op0);
3119 }
3120 return Register();
3121}
3122
3123Register fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3124 if (RetVT.SimpleTy != MVT::v2f64)
3125 return Register();
3126 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv2i32, RC: &AArch64::FPR128RegClass, Op0);
3127}
3128
3129Register fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3130 switch (VT.SimpleTy) {
3131 case MVT::f16: return fastEmit_ISD_FP_EXTEND_MVT_f16_r(RetVT, Op0);
3132 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
3133 case MVT::v4f16: return fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
3134 case MVT::v4bf16: return fastEmit_ISD_FP_EXTEND_MVT_v4bf16_r(RetVT, Op0);
3135 case MVT::v2f32: return fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(RetVT, Op0);
3136 default: return Register();
3137 }
3138}
3139
3140// FastEmit functions for ISD::FP_ROUND.
3141
3142Register fastEmit_ISD_FP_ROUND_MVT_f32_MVT_bf16_r(Register Op0) {
3143 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
3144 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVT, RC: &AArch64::FPR16RegClass, Op0);
3145 }
3146 return Register();
3147}
3148
3149Register fastEmit_ISD_FP_ROUND_MVT_f32_MVT_f16_r(Register Op0) {
3150 if ((Subtarget->hasFPARMv8())) {
3151 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHSr, RC: &AArch64::FPR16RegClass, Op0);
3152 }
3153 return Register();
3154}
3155
3156Register fastEmit_ISD_FP_ROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3157switch (RetVT.SimpleTy) {
3158 case MVT::bf16: return fastEmit_ISD_FP_ROUND_MVT_f32_MVT_bf16_r(Op0);
3159 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f32_MVT_f16_r(Op0);
3160 default: return Register();
3161}
3162}
3163
3164Register fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Register Op0) {
3165 if ((Subtarget->hasFPARMv8())) {
3166 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHDr, RC: &AArch64::FPR16RegClass, Op0);
3167 }
3168 return Register();
3169}
3170
3171Register fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Register Op0) {
3172 if ((Subtarget->hasFPARMv8())) {
3173 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSDr, RC: &AArch64::FPR32RegClass, Op0);
3174 }
3175 return Register();
3176}
3177
3178Register fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3179switch (RetVT.SimpleTy) {
3180 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Op0);
3181 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Op0);
3182 default: return Register();
3183}
3184}
3185
3186Register fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Register Op0) {
3187 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
3188}
3189
3190Register fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Register Op0) {
3191 if ((Subtarget->hasBF16()) && (Subtarget->isNeonAvailable())) {
3192 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVTN, RC: &AArch64::FPR64RegClass, Op0);
3193 }
3194 return Register();
3195}
3196
3197Register fastEmit_ISD_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3198switch (RetVT.SimpleTy) {
3199 case MVT::v4f16: return fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Op0);
3200 case MVT::v4bf16: return fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Op0);
3201 default: return Register();
3202}
3203}
3204
3205Register fastEmit_ISD_FP_ROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
3206 if (RetVT.SimpleTy != MVT::v2f32)
3207 return Register();
3208 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
3209}
3210
3211Register fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
3212 switch (VT.SimpleTy) {
3213 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f32_r(RetVT, Op0);
3214 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
3215 case MVT::v4f32: return fastEmit_ISD_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
3216 case MVT::v2f64: return fastEmit_ISD_FP_ROUND_MVT_v2f64_r(RetVT, Op0);
3217 default: return Register();
3218 }
3219}
3220
3221// FastEmit functions for ISD::FP_TO_SINT.
3222
3223Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
3224 if ((Subtarget->hasFPRCVT())) {
3225 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSHr, RC: &AArch64::FPR32RegClass, Op0);
3226 }
3227 if ((Subtarget->hasFullFP16())) {
3228 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
3229 }
3230 return Register();
3231}
3232
3233Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
3234 if ((Subtarget->hasFPRCVT())) {
3235 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDHr, RC: &AArch64::FPR64RegClass, Op0);
3236 }
3237 if ((Subtarget->hasFullFP16())) {
3238 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
3239 }
3240 return Register();
3241}
3242
3243Register fastEmit_ISD_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
3244switch (RetVT.SimpleTy) {
3245 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
3246 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
3247 default: return Register();
3248}
3249}
3250
3251Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
3252 if ((Subtarget->hasFPARMv8())) {
3253 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
3254 }
3255 return Register();
3256}
3257
3258Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
3259 if ((Subtarget->hasFPRCVT())) {
3260 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDSr, RC: &AArch64::FPR64RegClass, Op0);
3261 }
3262 if ((Subtarget->hasFPARMv8())) {
3263 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
3264 }
3265 return Register();
3266}
3267
3268Register fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
3269switch (RetVT.SimpleTy) {
3270 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
3271 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
3272 default: return Register();
3273}
3274}
3275
3276Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
3277 if ((Subtarget->hasFPRCVT())) {
3278 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSDr, RC: &AArch64::FPR32RegClass, Op0);
3279 }
3280 if ((Subtarget->hasFPARMv8())) {
3281 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
3282 }
3283 return Register();
3284}
3285
3286Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
3287 if ((Subtarget->hasFPARMv8())) {
3288 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
3289 }
3290 return Register();
3291}
3292
3293Register fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
3294switch (RetVT.SimpleTy) {
3295 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
3296 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
3297 default: return Register();
3298}
3299}
3300
3301Register fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3302 if (RetVT.SimpleTy != MVT::v4i16)
3303 return Register();
3304 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3305 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
3306 }
3307 return Register();
3308}
3309
3310Register fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3311 if (RetVT.SimpleTy != MVT::v8i16)
3312 return Register();
3313 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3314 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
3315 }
3316 return Register();
3317}
3318
3319Register fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3320 if (RetVT.SimpleTy != MVT::v2i32)
3321 return Register();
3322 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3323 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
3324 }
3325 return Register();
3326}
3327
3328Register fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3329 if (RetVT.SimpleTy != MVT::v4i32)
3330 return Register();
3331 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3332 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
3333 }
3334 return Register();
3335}
3336
3337Register fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3338 if (RetVT.SimpleTy != MVT::v2i64)
3339 return Register();
3340 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3341 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
3342 }
3343 return Register();
3344}
3345
3346Register fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
3347 switch (VT.SimpleTy) {
3348 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
3349 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
3350 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
3351 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
3352 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
3353 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
3354 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
3355 case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0);
3356 default: return Register();
3357 }
3358}
3359
3360// FastEmit functions for ISD::FP_TO_UINT.
3361
3362Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
3363 if ((Subtarget->hasFPRCVT())) {
3364 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSHr, RC: &AArch64::FPR32RegClass, Op0);
3365 }
3366 if ((Subtarget->hasFullFP16())) {
3367 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
3368 }
3369 return Register();
3370}
3371
3372Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
3373 if ((Subtarget->hasFPRCVT())) {
3374 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDHr, RC: &AArch64::FPR64RegClass, Op0);
3375 }
3376 if ((Subtarget->hasFullFP16())) {
3377 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
3378 }
3379 return Register();
3380}
3381
3382Register fastEmit_ISD_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
3383switch (RetVT.SimpleTy) {
3384 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
3385 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
3386 default: return Register();
3387}
3388}
3389
3390Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
3391 if ((Subtarget->hasFPARMv8())) {
3392 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
3393 }
3394 return Register();
3395}
3396
3397Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
3398 if ((Subtarget->hasFPRCVT())) {
3399 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDSr, RC: &AArch64::FPR64RegClass, Op0);
3400 }
3401 if ((Subtarget->hasFPARMv8())) {
3402 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
3403 }
3404 return Register();
3405}
3406
3407Register fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
3408switch (RetVT.SimpleTy) {
3409 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
3410 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
3411 default: return Register();
3412}
3413}
3414
3415Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
3416 if ((Subtarget->hasFPRCVT())) {
3417 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSDr, RC: &AArch64::FPR32RegClass, Op0);
3418 }
3419 if ((Subtarget->hasFPARMv8())) {
3420 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
3421 }
3422 return Register();
3423}
3424
3425Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
3426 if ((Subtarget->hasFPARMv8())) {
3427 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
3428 }
3429 return Register();
3430}
3431
3432Register fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
3433switch (RetVT.SimpleTy) {
3434 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
3435 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
3436 default: return Register();
3437}
3438}
3439
3440Register fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3441 if (RetVT.SimpleTy != MVT::v4i16)
3442 return Register();
3443 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3444 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
3445 }
3446 return Register();
3447}
3448
3449Register fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3450 if (RetVT.SimpleTy != MVT::v8i16)
3451 return Register();
3452 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3453 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
3454 }
3455 return Register();
3456}
3457
3458Register fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3459 if (RetVT.SimpleTy != MVT::v2i32)
3460 return Register();
3461 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3462 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
3463 }
3464 return Register();
3465}
3466
3467Register fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3468 if (RetVT.SimpleTy != MVT::v4i32)
3469 return Register();
3470 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3471 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
3472 }
3473 return Register();
3474}
3475
3476Register fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3477 if (RetVT.SimpleTy != MVT::v2i64)
3478 return Register();
3479 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3480 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
3481 }
3482 return Register();
3483}
3484
3485Register fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
3486 switch (VT.SimpleTy) {
3487 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
3488 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
3489 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
3490 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
3491 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
3492 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
3493 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
3494 case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0);
3495 default: return Register();
3496 }
3497}
3498
3499// FastEmit functions for ISD::FRINT.
3500
3501Register fastEmit_ISD_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
3502 if (RetVT.SimpleTy != MVT::f16)
3503 return Register();
3504 if ((Subtarget->hasFullFP16())) {
3505 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXHr, RC: &AArch64::FPR16RegClass, Op0);
3506 }
3507 return Register();
3508}
3509
3510Register fastEmit_ISD_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
3511 if (RetVT.SimpleTy != MVT::f32)
3512 return Register();
3513 if ((Subtarget->hasFPARMv8())) {
3514 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXSr, RC: &AArch64::FPR32RegClass, Op0);
3515 }
3516 return Register();
3517}
3518
3519Register fastEmit_ISD_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
3520 if (RetVT.SimpleTy != MVT::f64)
3521 return Register();
3522 if ((Subtarget->hasFPARMv8())) {
3523 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXDr, RC: &AArch64::FPR64RegClass, Op0);
3524 }
3525 return Register();
3526}
3527
3528Register fastEmit_ISD_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3529 if (RetVT.SimpleTy != MVT::v4f16)
3530 return Register();
3531 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3532 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f16, RC: &AArch64::FPR64RegClass, Op0);
3533 }
3534 return Register();
3535}
3536
3537Register fastEmit_ISD_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3538 if (RetVT.SimpleTy != MVT::v8f16)
3539 return Register();
3540 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3541 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv8f16, RC: &AArch64::FPR128RegClass, Op0);
3542 }
3543 return Register();
3544}
3545
3546Register fastEmit_ISD_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3547 if (RetVT.SimpleTy != MVT::v2f32)
3548 return Register();
3549 if ((Subtarget->isNeonAvailable())) {
3550 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f32, RC: &AArch64::FPR64RegClass, Op0);
3551 }
3552 return Register();
3553}
3554
3555Register fastEmit_ISD_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3556 if (RetVT.SimpleTy != MVT::v4f32)
3557 return Register();
3558 if ((Subtarget->isNeonAvailable())) {
3559 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f32, RC: &AArch64::FPR128RegClass, Op0);
3560 }
3561 return Register();
3562}
3563
3564Register fastEmit_ISD_FRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3565 if (RetVT.SimpleTy != MVT::v2f64)
3566 return Register();
3567 if ((Subtarget->isNeonAvailable())) {
3568 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f64, RC: &AArch64::FPR128RegClass, Op0);
3569 }
3570 return Register();
3571}
3572
3573Register fastEmit_ISD_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
3574 switch (VT.SimpleTy) {
3575 case MVT::f16: return fastEmit_ISD_FRINT_MVT_f16_r(RetVT, Op0);
3576 case MVT::f32: return fastEmit_ISD_FRINT_MVT_f32_r(RetVT, Op0);
3577 case MVT::f64: return fastEmit_ISD_FRINT_MVT_f64_r(RetVT, Op0);
3578 case MVT::v4f16: return fastEmit_ISD_FRINT_MVT_v4f16_r(RetVT, Op0);
3579 case MVT::v8f16: return fastEmit_ISD_FRINT_MVT_v8f16_r(RetVT, Op0);
3580 case MVT::v2f32: return fastEmit_ISD_FRINT_MVT_v2f32_r(RetVT, Op0);
3581 case MVT::v4f32: return fastEmit_ISD_FRINT_MVT_v4f32_r(RetVT, Op0);
3582 case MVT::v2f64: return fastEmit_ISD_FRINT_MVT_v2f64_r(RetVT, Op0);
3583 default: return Register();
3584 }
3585}
3586
3587// FastEmit functions for ISD::FROUND.
3588
3589Register fastEmit_ISD_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
3590 if (RetVT.SimpleTy != MVT::f16)
3591 return Register();
3592 if ((Subtarget->hasFullFP16())) {
3593 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAHr, RC: &AArch64::FPR16RegClass, Op0);
3594 }
3595 return Register();
3596}
3597
3598Register fastEmit_ISD_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3599 if (RetVT.SimpleTy != MVT::f32)
3600 return Register();
3601 if ((Subtarget->hasFPARMv8())) {
3602 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTASr, RC: &AArch64::FPR32RegClass, Op0);
3603 }
3604 return Register();
3605}
3606
3607Register fastEmit_ISD_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3608 if (RetVT.SimpleTy != MVT::f64)
3609 return Register();
3610 if ((Subtarget->hasFPARMv8())) {
3611 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTADr, RC: &AArch64::FPR64RegClass, Op0);
3612 }
3613 return Register();
3614}
3615
3616Register fastEmit_ISD_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3617 if (RetVT.SimpleTy != MVT::v4f16)
3618 return Register();
3619 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3620 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f16, RC: &AArch64::FPR64RegClass, Op0);
3621 }
3622 return Register();
3623}
3624
3625Register fastEmit_ISD_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
3626 if (RetVT.SimpleTy != MVT::v8f16)
3627 return Register();
3628 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3629 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv8f16, RC: &AArch64::FPR128RegClass, Op0);
3630 }
3631 return Register();
3632}
3633
3634Register fastEmit_ISD_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3635 if (RetVT.SimpleTy != MVT::v2f32)
3636 return Register();
3637 if ((Subtarget->isNeonAvailable())) {
3638 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f32, RC: &AArch64::FPR64RegClass, Op0);
3639 }
3640 return Register();
3641}
3642
3643Register fastEmit_ISD_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3644 if (RetVT.SimpleTy != MVT::v4f32)
3645 return Register();
3646 if ((Subtarget->isNeonAvailable())) {
3647 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f32, RC: &AArch64::FPR128RegClass, Op0);
3648 }
3649 return Register();
3650}
3651
3652Register fastEmit_ISD_FROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
3653 if (RetVT.SimpleTy != MVT::v2f64)
3654 return Register();
3655 if ((Subtarget->isNeonAvailable())) {
3656 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f64, RC: &AArch64::FPR128RegClass, Op0);
3657 }
3658 return Register();
3659}
3660
3661Register fastEmit_ISD_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
3662 switch (VT.SimpleTy) {
3663 case MVT::f16: return fastEmit_ISD_FROUND_MVT_f16_r(RetVT, Op0);
3664 case MVT::f32: return fastEmit_ISD_FROUND_MVT_f32_r(RetVT, Op0);
3665 case MVT::f64: return fastEmit_ISD_FROUND_MVT_f64_r(RetVT, Op0);
3666 case MVT::v4f16: return fastEmit_ISD_FROUND_MVT_v4f16_r(RetVT, Op0);
3667 case MVT::v8f16: return fastEmit_ISD_FROUND_MVT_v8f16_r(RetVT, Op0);
3668 case MVT::v2f32: return fastEmit_ISD_FROUND_MVT_v2f32_r(RetVT, Op0);
3669 case MVT::v4f32: return fastEmit_ISD_FROUND_MVT_v4f32_r(RetVT, Op0);
3670 case MVT::v2f64: return fastEmit_ISD_FROUND_MVT_v2f64_r(RetVT, Op0);
3671 default: return Register();
3672 }
3673}
3674
3675// FastEmit functions for ISD::FROUNDEVEN.
3676
3677Register fastEmit_ISD_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
3678 if (RetVT.SimpleTy != MVT::f16)
3679 return Register();
3680 if ((Subtarget->hasFullFP16())) {
3681 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNHr, RC: &AArch64::FPR16RegClass, Op0);
3682 }
3683 return Register();
3684}
3685
3686Register fastEmit_ISD_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
3687 if (RetVT.SimpleTy != MVT::f32)
3688 return Register();
3689 if ((Subtarget->hasFPARMv8())) {
3690 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNSr, RC: &AArch64::FPR32RegClass, Op0);
3691 }
3692 return Register();
3693}
3694
3695Register fastEmit_ISD_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
3696 if (RetVT.SimpleTy != MVT::f64)
3697 return Register();
3698 if ((Subtarget->hasFPARMv8())) {
3699 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNDr, RC: &AArch64::FPR64RegClass, Op0);
3700 }
3701 return Register();
3702}
3703
3704Register fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
3705 if (RetVT.SimpleTy != MVT::v4f16)
3706 return Register();
3707 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3708 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f16, RC: &AArch64::FPR64RegClass, Op0);
3709 }
3710 return Register();
3711}
3712
3713Register fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
3714 if (RetVT.SimpleTy != MVT::v8f16)
3715 return Register();
3716 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3717 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv8f16, RC: &AArch64::FPR128RegClass, Op0);
3718 }
3719 return Register();
3720}
3721
3722Register fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
3723 if (RetVT.SimpleTy != MVT::v2f32)
3724 return Register();
3725 if ((Subtarget->isNeonAvailable())) {
3726 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f32, RC: &AArch64::FPR64RegClass, Op0);
3727 }
3728 return Register();
3729}
3730
3731Register fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
3732 if (RetVT.SimpleTy != MVT::v4f32)
3733 return Register();
3734 if ((Subtarget->isNeonAvailable())) {
3735 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f32, RC: &AArch64::FPR128RegClass, Op0);
3736 }
3737 return Register();
3738}
3739
3740Register fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, Register Op0) {
3741 if (RetVT.SimpleTy != MVT::v2f64)
3742 return Register();
3743 if ((Subtarget->isNeonAvailable())) {
3744 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f64, RC: &AArch64::FPR128RegClass, Op0);
3745 }
3746 return Register();
3747}
3748
3749Register fastEmit_ISD_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
3750 switch (VT.SimpleTy) {
3751 case MVT::f16: return fastEmit_ISD_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
3752 case MVT::f32: return fastEmit_ISD_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
3753 case MVT::f64: return fastEmit_ISD_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
3754 case MVT::v4f16: return fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
3755 case MVT::v8f16: return fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
3756 case MVT::v2f32: return fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
3757 case MVT::v4f32: return fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
3758 case MVT::v2f64: return fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0);
3759 default: return Register();
3760 }
3761}
3762
3763// FastEmit functions for ISD::FSQRT.
3764
3765Register fastEmit_ISD_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
3766 if (RetVT.SimpleTy != MVT::f16)
3767 return Register();
3768 if ((Subtarget->hasFullFP16())) {
3769 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTHr, RC: &AArch64::FPR16RegClass, Op0);
3770 }
3771 return Register();
3772}
3773
3774Register fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
3775 if (RetVT.SimpleTy != MVT::f32)
3776 return Register();
3777 if ((Subtarget->hasFPARMv8())) {
3778 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTSr, RC: &AArch64::FPR32RegClass, Op0);
3779 }
3780 return Register();
3781}
3782
3783Register fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
3784 if (RetVT.SimpleTy != MVT::f64)
3785 return Register();
3786 if ((Subtarget->hasFPARMv8())) {
3787 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTDr, RC: &AArch64::FPR64RegClass, Op0);
3788 }
3789 return Register();
3790}
3791
3792Register fastEmit_ISD_FSQRT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3793 if (RetVT.SimpleTy != MVT::v4f16)
3794 return Register();
3795 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3796 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f16, RC: &AArch64::FPR64RegClass, Op0);
3797 }
3798 return Register();
3799}
3800
3801Register fastEmit_ISD_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3802 if (RetVT.SimpleTy != MVT::v8f16)
3803 return Register();
3804 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3805 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv8f16, RC: &AArch64::FPR128RegClass, Op0);
3806 }
3807 return Register();
3808}
3809
3810Register fastEmit_ISD_FSQRT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3811 if (RetVT.SimpleTy != MVT::v2f32)
3812 return Register();
3813 if ((Subtarget->isNeonAvailable())) {
3814 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f32, RC: &AArch64::FPR64RegClass, Op0);
3815 }
3816 return Register();
3817}
3818
3819Register fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3820 if (RetVT.SimpleTy != MVT::v4f32)
3821 return Register();
3822 if ((Subtarget->isNeonAvailable())) {
3823 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f32, RC: &AArch64::FPR128RegClass, Op0);
3824 }
3825 return Register();
3826}
3827
3828Register fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3829 if (RetVT.SimpleTy != MVT::v2f64)
3830 return Register();
3831 if ((Subtarget->isNeonAvailable())) {
3832 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f64, RC: &AArch64::FPR128RegClass, Op0);
3833 }
3834 return Register();
3835}
3836
3837Register fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
3838 switch (VT.SimpleTy) {
3839 case MVT::f16: return fastEmit_ISD_FSQRT_MVT_f16_r(RetVT, Op0);
3840 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
3841 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
3842 case MVT::v4f16: return fastEmit_ISD_FSQRT_MVT_v4f16_r(RetVT, Op0);
3843 case MVT::v8f16: return fastEmit_ISD_FSQRT_MVT_v8f16_r(RetVT, Op0);
3844 case MVT::v2f32: return fastEmit_ISD_FSQRT_MVT_v2f32_r(RetVT, Op0);
3845 case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0);
3846 case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0);
3847 default: return Register();
3848 }
3849}
3850
3851// FastEmit functions for ISD::FTRUNC.
3852
3853Register fastEmit_ISD_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
3854 if (RetVT.SimpleTy != MVT::f16)
3855 return Register();
3856 if ((Subtarget->hasFullFP16())) {
3857 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZHr, RC: &AArch64::FPR16RegClass, Op0);
3858 }
3859 return Register();
3860}
3861
3862Register fastEmit_ISD_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
3863 if (RetVT.SimpleTy != MVT::f32)
3864 return Register();
3865 if ((Subtarget->hasFPARMv8())) {
3866 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZSr, RC: &AArch64::FPR32RegClass, Op0);
3867 }
3868 return Register();
3869}
3870
3871Register fastEmit_ISD_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
3872 if (RetVT.SimpleTy != MVT::f64)
3873 return Register();
3874 if ((Subtarget->hasFPARMv8())) {
3875 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZDr, RC: &AArch64::FPR64RegClass, Op0);
3876 }
3877 return Register();
3878}
3879
3880Register fastEmit_ISD_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
3881 if (RetVT.SimpleTy != MVT::v4f16)
3882 return Register();
3883 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3884 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f16, RC: &AArch64::FPR64RegClass, Op0);
3885 }
3886 return Register();
3887}
3888
3889Register fastEmit_ISD_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
3890 if (RetVT.SimpleTy != MVT::v8f16)
3891 return Register();
3892 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3893 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv8f16, RC: &AArch64::FPR128RegClass, Op0);
3894 }
3895 return Register();
3896}
3897
3898Register fastEmit_ISD_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
3899 if (RetVT.SimpleTy != MVT::v2f32)
3900 return Register();
3901 if ((Subtarget->isNeonAvailable())) {
3902 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f32, RC: &AArch64::FPR64RegClass, Op0);
3903 }
3904 return Register();
3905}
3906
3907Register fastEmit_ISD_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
3908 if (RetVT.SimpleTy != MVT::v4f32)
3909 return Register();
3910 if ((Subtarget->isNeonAvailable())) {
3911 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f32, RC: &AArch64::FPR128RegClass, Op0);
3912 }
3913 return Register();
3914}
3915
3916Register fastEmit_ISD_FTRUNC_MVT_v2f64_r(MVT RetVT, Register Op0) {
3917 if (RetVT.SimpleTy != MVT::v2f64)
3918 return Register();
3919 if ((Subtarget->isNeonAvailable())) {
3920 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f64, RC: &AArch64::FPR128RegClass, Op0);
3921 }
3922 return Register();
3923}
3924
3925Register fastEmit_ISD_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
3926 switch (VT.SimpleTy) {
3927 case MVT::f16: return fastEmit_ISD_FTRUNC_MVT_f16_r(RetVT, Op0);
3928 case MVT::f32: return fastEmit_ISD_FTRUNC_MVT_f32_r(RetVT, Op0);
3929 case MVT::f64: return fastEmit_ISD_FTRUNC_MVT_f64_r(RetVT, Op0);
3930 case MVT::v4f16: return fastEmit_ISD_FTRUNC_MVT_v4f16_r(RetVT, Op0);
3931 case MVT::v8f16: return fastEmit_ISD_FTRUNC_MVT_v8f16_r(RetVT, Op0);
3932 case MVT::v2f32: return fastEmit_ISD_FTRUNC_MVT_v2f32_r(RetVT, Op0);
3933 case MVT::v4f32: return fastEmit_ISD_FTRUNC_MVT_v4f32_r(RetVT, Op0);
3934 case MVT::v2f64: return fastEmit_ISD_FTRUNC_MVT_v2f64_r(RetVT, Op0);
3935 default: return Register();
3936 }
3937}
3938
3939// FastEmit functions for ISD::LLROUND.
3940
3941Register fastEmit_ISD_LLROUND_MVT_f16_r(MVT RetVT, Register Op0) {
3942 if (RetVT.SimpleTy != MVT::i64)
3943 return Register();
3944 if ((Subtarget->hasFPRCVT())) {
3945 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDHr, RC: &AArch64::FPR64RegClass, Op0);
3946 }
3947 if ((Subtarget->hasFullFP16())) {
3948 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
3949 }
3950 return Register();
3951}
3952
3953Register fastEmit_ISD_LLROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3954 if (RetVT.SimpleTy != MVT::i64)
3955 return Register();
3956 if ((Subtarget->hasFPRCVT())) {
3957 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDSr, RC: &AArch64::FPR64RegClass, Op0);
3958 }
3959 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
3960}
3961
3962Register fastEmit_ISD_LLROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3963 if (RetVT.SimpleTy != MVT::i64)
3964 return Register();
3965 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
3966}
3967
3968Register fastEmit_ISD_LLROUND_r(MVT VT, MVT RetVT, Register Op0) {
3969 switch (VT.SimpleTy) {
3970 case MVT::f16: return fastEmit_ISD_LLROUND_MVT_f16_r(RetVT, Op0);
3971 case MVT::f32: return fastEmit_ISD_LLROUND_MVT_f32_r(RetVT, Op0);
3972 case MVT::f64: return fastEmit_ISD_LLROUND_MVT_f64_r(RetVT, Op0);
3973 default: return Register();
3974 }
3975}
3976
3977// FastEmit functions for ISD::LROUND.
3978
3979Register fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(Register Op0) {
3980 if ((Subtarget->hasFPRCVT())) {
3981 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASSHr, RC: &AArch64::FPR32RegClass, Op0);
3982 }
3983 if ((Subtarget->hasFullFP16())) {
3984 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWHr, RC: &AArch64::GPR32RegClass, Op0);
3985 }
3986 return Register();
3987}
3988
3989Register fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(Register Op0) {
3990 if ((Subtarget->hasFPRCVT())) {
3991 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDHr, RC: &AArch64::FPR64RegClass, Op0);
3992 }
3993 if ((Subtarget->hasFullFP16())) {
3994 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
3995 }
3996 return Register();
3997}
3998
3999Register fastEmit_ISD_LROUND_MVT_f16_r(MVT RetVT, Register Op0) {
4000switch (RetVT.SimpleTy) {
4001 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(Op0);
4002 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(Op0);
4003 default: return Register();
4004}
4005}
4006
4007Register fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(Register Op0) {
4008 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWSr, RC: &AArch64::GPR32RegClass, Op0);
4009}
4010
4011Register fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(Register Op0) {
4012 if ((Subtarget->hasFPRCVT())) {
4013 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDSr, RC: &AArch64::FPR64RegClass, Op0);
4014 }
4015 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
4016}
4017
4018Register fastEmit_ISD_LROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4019switch (RetVT.SimpleTy) {
4020 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(Op0);
4021 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(Op0);
4022 default: return Register();
4023}
4024}
4025
4026Register fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(Register Op0) {
4027 if ((Subtarget->hasFPRCVT())) {
4028 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASSDr, RC: &AArch64::FPR32RegClass, Op0);
4029 }
4030 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWDr, RC: &AArch64::GPR32RegClass, Op0);
4031}
4032
4033Register fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(Register Op0) {
4034 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
4035}
4036
4037Register fastEmit_ISD_LROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4038switch (RetVT.SimpleTy) {
4039 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(Op0);
4040 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(Op0);
4041 default: return Register();
4042}
4043}
4044
4045Register fastEmit_ISD_LROUND_r(MVT VT, MVT RetVT, Register Op0) {
4046 switch (VT.SimpleTy) {
4047 case MVT::f16: return fastEmit_ISD_LROUND_MVT_f16_r(RetVT, Op0);
4048 case MVT::f32: return fastEmit_ISD_LROUND_MVT_f32_r(RetVT, Op0);
4049 case MVT::f64: return fastEmit_ISD_LROUND_MVT_f64_r(RetVT, Op0);
4050 default: return Register();
4051 }
4052}
4053
4054// FastEmit functions for ISD::SINT_TO_FP.
4055
4056Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
4057 if ((Subtarget->hasFPRCVT())) {
4058 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
4059 }
4060 if ((Subtarget->hasFullFP16())) {
4061 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
4062 }
4063 return Register();
4064}
4065
4066Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
4067 if ((Subtarget->hasFPARMv8())) {
4068 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
4069 }
4070 return Register();
4071}
4072
4073Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
4074 if ((Subtarget->hasFPRCVT())) {
4075 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
4076 }
4077 if ((Subtarget->hasFPARMv8())) {
4078 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
4079 }
4080 return Register();
4081}
4082
4083Register fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
4084switch (RetVT.SimpleTy) {
4085 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
4086 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
4087 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
4088 default: return Register();
4089}
4090}
4091
4092Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
4093 if ((Subtarget->hasFPRCVT())) {
4094 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
4095 }
4096 if ((Subtarget->hasFullFP16())) {
4097 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
4098 }
4099 return Register();
4100}
4101
4102Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
4103 if ((Subtarget->hasFPRCVT())) {
4104 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
4105 }
4106 if ((Subtarget->hasFPARMv8())) {
4107 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
4108 }
4109 return Register();
4110}
4111
4112Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
4113 if ((Subtarget->hasFPARMv8())) {
4114 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
4115 }
4116 return Register();
4117}
4118
4119Register fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
4120switch (RetVT.SimpleTy) {
4121 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
4122 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
4123 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
4124 default: return Register();
4125}
4126}
4127
4128Register fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
4129 if (RetVT.SimpleTy != MVT::v4f16)
4130 return Register();
4131 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4132 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
4133 }
4134 return Register();
4135}
4136
4137Register fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
4138 if (RetVT.SimpleTy != MVT::v8f16)
4139 return Register();
4140 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4141 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
4142 }
4143 return Register();
4144}
4145
4146Register fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
4147 if (RetVT.SimpleTy != MVT::v2f32)
4148 return Register();
4149 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4150 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
4151 }
4152 return Register();
4153}
4154
4155Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
4156 if (RetVT.SimpleTy != MVT::v4f32)
4157 return Register();
4158 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4159 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
4160 }
4161 return Register();
4162}
4163
4164Register fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
4165 if (RetVT.SimpleTy != MVT::v2f64)
4166 return Register();
4167 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4168 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
4169 }
4170 return Register();
4171}
4172
4173Register fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
4174 switch (VT.SimpleTy) {
4175 case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
4176 case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
4177 case MVT::v4i16: return fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
4178 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
4179 case MVT::v2i32: return fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
4180 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
4181 case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
4182 default: return Register();
4183 }
4184}
4185
4186// FastEmit functions for ISD::SPLAT_VECTOR.
4187
4188Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(Register Op0) {
4189 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4190 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_B, RC: &AArch64::ZPRRegClass, Op0);
4191 }
4192 return Register();
4193}
4194
4195Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(Register Op0) {
4196 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4197 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_H, RC: &AArch64::ZPRRegClass, Op0);
4198 }
4199 return Register();
4200}
4201
4202Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(Register Op0) {
4203 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4204 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_S, RC: &AArch64::ZPRRegClass, Op0);
4205 }
4206 return Register();
4207}
4208
4209Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(MVT RetVT, Register Op0) {
4210switch (RetVT.SimpleTy) {
4211 case MVT::nxv16i8: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(Op0);
4212 case MVT::nxv8i16: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(Op0);
4213 case MVT::nxv4i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(Op0);
4214 default: return Register();
4215}
4216}
4217
4218Register fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(MVT RetVT, Register Op0) {
4219 if (RetVT.SimpleTy != MVT::nxv2i64)
4220 return Register();
4221 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4222 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_D, RC: &AArch64::ZPRRegClass, Op0);
4223 }
4224 return Register();
4225}
4226
4227Register fastEmit_ISD_SPLAT_VECTOR_r(MVT VT, MVT RetVT, Register Op0) {
4228 switch (VT.SimpleTy) {
4229 case MVT::i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(RetVT, Op0);
4230 case MVT::i64: return fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(RetVT, Op0);
4231 default: return Register();
4232 }
4233}
4234
4235// FastEmit functions for ISD::STRICT_FCEIL.
4236
4237Register fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
4238 if (RetVT.SimpleTy != MVT::f16)
4239 return Register();
4240 if ((Subtarget->hasFullFP16())) {
4241 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPHr, RC: &AArch64::FPR16RegClass, Op0);
4242 }
4243 return Register();
4244}
4245
4246Register fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
4247 if (RetVT.SimpleTy != MVT::f32)
4248 return Register();
4249 if ((Subtarget->hasFPARMv8())) {
4250 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPSr, RC: &AArch64::FPR32RegClass, Op0);
4251 }
4252 return Register();
4253}
4254
4255Register fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
4256 if (RetVT.SimpleTy != MVT::f64)
4257 return Register();
4258 if ((Subtarget->hasFPARMv8())) {
4259 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPDr, RC: &AArch64::FPR64RegClass, Op0);
4260 }
4261 return Register();
4262}
4263
4264Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
4265 if (RetVT.SimpleTy != MVT::v4f16)
4266 return Register();
4267 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4268 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f16, RC: &AArch64::FPR64RegClass, Op0);
4269 }
4270 return Register();
4271}
4272
4273Register fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
4274 if (RetVT.SimpleTy != MVT::v8f16)
4275 return Register();
4276 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4277 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv8f16, RC: &AArch64::FPR128RegClass, Op0);
4278 }
4279 return Register();
4280}
4281
4282Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
4283 if (RetVT.SimpleTy != MVT::v2f32)
4284 return Register();
4285 if ((Subtarget->isNeonAvailable())) {
4286 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f32, RC: &AArch64::FPR64RegClass, Op0);
4287 }
4288 return Register();
4289}
4290
4291Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
4292 if (RetVT.SimpleTy != MVT::v4f32)
4293 return Register();
4294 if ((Subtarget->isNeonAvailable())) {
4295 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f32, RC: &AArch64::FPR128RegClass, Op0);
4296 }
4297 return Register();
4298}
4299
4300Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(MVT RetVT, Register Op0) {
4301 if (RetVT.SimpleTy != MVT::v2f64)
4302 return Register();
4303 if ((Subtarget->isNeonAvailable())) {
4304 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f64, RC: &AArch64::FPR128RegClass, Op0);
4305 }
4306 return Register();
4307}
4308
4309Register fastEmit_ISD_STRICT_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
4310 switch (VT.SimpleTy) {
4311 case MVT::f16: return fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(RetVT, Op0);
4312 case MVT::f32: return fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(RetVT, Op0);
4313 case MVT::f64: return fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(RetVT, Op0);
4314 case MVT::v4f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(RetVT, Op0);
4315 case MVT::v8f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(RetVT, Op0);
4316 case MVT::v2f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(RetVT, Op0);
4317 case MVT::v4f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(RetVT, Op0);
4318 case MVT::v2f64: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(RetVT, Op0);
4319 default: return Register();
4320 }
4321}
4322
4323// FastEmit functions for ISD::STRICT_FFLOOR.
4324
4325Register fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
4326 if (RetVT.SimpleTy != MVT::f16)
4327 return Register();
4328 if ((Subtarget->hasFullFP16())) {
4329 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMHr, RC: &AArch64::FPR16RegClass, Op0);
4330 }
4331 return Register();
4332}
4333
4334Register fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
4335 if (RetVT.SimpleTy != MVT::f32)
4336 return Register();
4337 if ((Subtarget->hasFPARMv8())) {
4338 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMSr, RC: &AArch64::FPR32RegClass, Op0);
4339 }
4340 return Register();
4341}
4342
4343Register fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
4344 if (RetVT.SimpleTy != MVT::f64)
4345 return Register();
4346 if ((Subtarget->hasFPARMv8())) {
4347 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMDr, RC: &AArch64::FPR64RegClass, Op0);
4348 }
4349 return Register();
4350}
4351
4352Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
4353 if (RetVT.SimpleTy != MVT::v4f16)
4354 return Register();
4355 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4356 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f16, RC: &AArch64::FPR64RegClass, Op0);
4357 }
4358 return Register();
4359}
4360
4361Register fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
4362 if (RetVT.SimpleTy != MVT::v8f16)
4363 return Register();
4364 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4365 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv8f16, RC: &AArch64::FPR128RegClass, Op0);
4366 }
4367 return Register();
4368}
4369
4370Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
4371 if (RetVT.SimpleTy != MVT::v2f32)
4372 return Register();
4373 if ((Subtarget->isNeonAvailable())) {
4374 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f32, RC: &AArch64::FPR64RegClass, Op0);
4375 }
4376 return Register();
4377}
4378
4379Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
4380 if (RetVT.SimpleTy != MVT::v4f32)
4381 return Register();
4382 if ((Subtarget->isNeonAvailable())) {
4383 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f32, RC: &AArch64::FPR128RegClass, Op0);
4384 }
4385 return Register();
4386}
4387
4388Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(MVT RetVT, Register Op0) {
4389 if (RetVT.SimpleTy != MVT::v2f64)
4390 return Register();
4391 if ((Subtarget->isNeonAvailable())) {
4392 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f64, RC: &AArch64::FPR128RegClass, Op0);
4393 }
4394 return Register();
4395}
4396
4397Register fastEmit_ISD_STRICT_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
4398 switch (VT.SimpleTy) {
4399 case MVT::f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(RetVT, Op0);
4400 case MVT::f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(RetVT, Op0);
4401 case MVT::f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(RetVT, Op0);
4402 case MVT::v4f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(RetVT, Op0);
4403 case MVT::v8f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(RetVT, Op0);
4404 case MVT::v2f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(RetVT, Op0);
4405 case MVT::v4f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(RetVT, Op0);
4406 case MVT::v2f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(RetVT, Op0);
4407 default: return Register();
4408 }
4409}
4410
4411// FastEmit functions for ISD::STRICT_FNEARBYINT.
4412
4413Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
4414 if (RetVT.SimpleTy != MVT::f16)
4415 return Register();
4416 if ((Subtarget->hasFullFP16())) {
4417 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIHr, RC: &AArch64::FPR16RegClass, Op0);
4418 }
4419 return Register();
4420}
4421
4422Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
4423 if (RetVT.SimpleTy != MVT::f32)
4424 return Register();
4425 if ((Subtarget->hasFPARMv8())) {
4426 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTISr, RC: &AArch64::FPR32RegClass, Op0);
4427 }
4428 return Register();
4429}
4430
4431Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
4432 if (RetVT.SimpleTy != MVT::f64)
4433 return Register();
4434 if ((Subtarget->hasFPARMv8())) {
4435 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIDr, RC: &AArch64::FPR64RegClass, Op0);
4436 }
4437 return Register();
4438}
4439
4440Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4441 if (RetVT.SimpleTy != MVT::v4f16)
4442 return Register();
4443 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4444 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f16, RC: &AArch64::FPR64RegClass, Op0);
4445 }
4446 return Register();
4447}
4448
4449Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4450 if (RetVT.SimpleTy != MVT::v8f16)
4451 return Register();
4452 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4453 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv8f16, RC: &AArch64::FPR128RegClass, Op0);
4454 }
4455 return Register();
4456}
4457
4458Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4459 if (RetVT.SimpleTy != MVT::v2f32)
4460 return Register();
4461 if ((Subtarget->isNeonAvailable())) {
4462 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f32, RC: &AArch64::FPR64RegClass, Op0);
4463 }
4464 return Register();
4465}
4466
4467Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4468 if (RetVT.SimpleTy != MVT::v4f32)
4469 return Register();
4470 if ((Subtarget->isNeonAvailable())) {
4471 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f32, RC: &AArch64::FPR128RegClass, Op0);
4472 }
4473 return Register();
4474}
4475
4476Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4477 if (RetVT.SimpleTy != MVT::v2f64)
4478 return Register();
4479 if ((Subtarget->isNeonAvailable())) {
4480 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f64, RC: &AArch64::FPR128RegClass, Op0);
4481 }
4482 return Register();
4483}
4484
4485Register fastEmit_ISD_STRICT_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
4486 switch (VT.SimpleTy) {
4487 case MVT::f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(RetVT, Op0);
4488 case MVT::f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(RetVT, Op0);
4489 case MVT::f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(RetVT, Op0);
4490 case MVT::v4f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(RetVT, Op0);
4491 case MVT::v8f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(RetVT, Op0);
4492 case MVT::v2f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(RetVT, Op0);
4493 case MVT::v4f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(RetVT, Op0);
4494 case MVT::v2f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(RetVT, Op0);
4495 default: return Register();
4496 }
4497}
4498
4499// FastEmit functions for ISD::STRICT_FP_EXTEND.
4500
4501Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(Register Op0) {
4502 if ((Subtarget->hasFPARMv8())) {
4503 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSHr, RC: &AArch64::FPR32RegClass, Op0);
4504 }
4505 return Register();
4506}
4507
4508Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(Register Op0) {
4509 if ((Subtarget->hasFPARMv8())) {
4510 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDHr, RC: &AArch64::FPR64RegClass, Op0);
4511 }
4512 return Register();
4513}
4514
4515Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(MVT RetVT, Register Op0) {
4516switch (RetVT.SimpleTy) {
4517 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(Op0);
4518 case MVT::f64: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(Op0);
4519 default: return Register();
4520}
4521}
4522
4523Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
4524 if (RetVT.SimpleTy != MVT::f64)
4525 return Register();
4526 if ((Subtarget->hasFPARMv8())) {
4527 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDSr, RC: &AArch64::FPR64RegClass, Op0);
4528 }
4529 return Register();
4530}
4531
4532Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
4533 if (RetVT.SimpleTy != MVT::v4f32)
4534 return Register();
4535 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv4i16, RC: &AArch64::FPR128RegClass, Op0);
4536}
4537
4538Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4bf16_r(MVT RetVT, Register Op0) {
4539 if (RetVT.SimpleTy != MVT::v4f32)
4540 return Register();
4541 if ((Subtarget->isNeonAvailable())) {
4542 return fastEmitInst_r(MachineInstOpcode: AArch64::SHLLv4i16, RC: &AArch64::FPR128RegClass, Op0);
4543 }
4544 return Register();
4545}
4546
4547Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(MVT RetVT, Register Op0) {
4548 if (RetVT.SimpleTy != MVT::v2f64)
4549 return Register();
4550 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv2i32, RC: &AArch64::FPR128RegClass, Op0);
4551}
4552
4553Register fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
4554 switch (VT.SimpleTy) {
4555 case MVT::f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(RetVT, Op0);
4556 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0);
4557 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
4558 case MVT::v4bf16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4bf16_r(RetVT, Op0);
4559 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(RetVT, Op0);
4560 default: return Register();
4561 }
4562}
4563
4564// FastEmit functions for ISD::STRICT_FP_ROUND.
4565
4566Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_bf16_r(Register Op0) {
4567 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
4568 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVT, RC: &AArch64::FPR16RegClass, Op0);
4569 }
4570 return Register();
4571}
4572
4573Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_f16_r(Register Op0) {
4574 if ((Subtarget->hasFPARMv8())) {
4575 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHSr, RC: &AArch64::FPR16RegClass, Op0);
4576 }
4577 return Register();
4578}
4579
4580Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4581switch (RetVT.SimpleTy) {
4582 case MVT::bf16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_bf16_r(Op0);
4583 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_f16_r(Op0);
4584 default: return Register();
4585}
4586}
4587
4588Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(Register Op0) {
4589 if ((Subtarget->hasFPARMv8())) {
4590 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHDr, RC: &AArch64::FPR16RegClass, Op0);
4591 }
4592 return Register();
4593}
4594
4595Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(Register Op0) {
4596 if ((Subtarget->hasFPARMv8())) {
4597 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSDr, RC: &AArch64::FPR32RegClass, Op0);
4598 }
4599 return Register();
4600}
4601
4602Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4603switch (RetVT.SimpleTy) {
4604 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(Op0);
4605 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(Op0);
4606 default: return Register();
4607}
4608}
4609
4610Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Register Op0) {
4611 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
4612}
4613
4614Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Register Op0) {
4615 if ((Subtarget->hasBF16()) && (Subtarget->isNeonAvailable())) {
4616 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVTN, RC: &AArch64::FPR64RegClass, Op0);
4617 }
4618 return Register();
4619}
4620
4621Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
4622switch (RetVT.SimpleTy) {
4623 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Op0);
4624 case MVT::v4bf16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Op0);
4625 default: return Register();
4626}
4627}
4628
4629Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
4630 if (RetVT.SimpleTy != MVT::v2f32)
4631 return Register();
4632 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
4633}
4634
4635Register fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
4636 switch (VT.SimpleTy) {
4637 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(RetVT, Op0);
4638 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0);
4639 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
4640 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(RetVT, Op0);
4641 default: return Register();
4642 }
4643}
4644
4645// FastEmit functions for ISD::STRICT_FP_TO_SINT.
4646
4647Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
4648 if ((Subtarget->hasFPRCVT())) {
4649 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSHr, RC: &AArch64::FPR32RegClass, Op0);
4650 }
4651 if ((Subtarget->hasFullFP16())) {
4652 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
4653 }
4654 return Register();
4655}
4656
4657Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
4658 if ((Subtarget->hasFPRCVT())) {
4659 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDHr, RC: &AArch64::FPR64RegClass, Op0);
4660 }
4661 if ((Subtarget->hasFullFP16())) {
4662 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
4663 }
4664 return Register();
4665}
4666
4667Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
4668switch (RetVT.SimpleTy) {
4669 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
4670 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
4671 default: return Register();
4672}
4673}
4674
4675Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
4676 if ((Subtarget->hasFPARMv8())) {
4677 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
4678 }
4679 return Register();
4680}
4681
4682Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
4683 if ((Subtarget->hasFPRCVT())) {
4684 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDSr, RC: &AArch64::FPR64RegClass, Op0);
4685 }
4686 if ((Subtarget->hasFPARMv8())) {
4687 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
4688 }
4689 return Register();
4690}
4691
4692Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
4693switch (RetVT.SimpleTy) {
4694 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
4695 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
4696 default: return Register();
4697}
4698}
4699
4700Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
4701 if ((Subtarget->hasFPRCVT())) {
4702 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSDr, RC: &AArch64::FPR32RegClass, Op0);
4703 }
4704 if ((Subtarget->hasFPARMv8())) {
4705 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
4706 }
4707 return Register();
4708}
4709
4710Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
4711 if ((Subtarget->hasFPARMv8())) {
4712 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
4713 }
4714 return Register();
4715}
4716
4717Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
4718switch (RetVT.SimpleTy) {
4719 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
4720 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
4721 default: return Register();
4722}
4723}
4724
4725Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4726 if (RetVT.SimpleTy != MVT::v4i16)
4727 return Register();
4728 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4729 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
4730 }
4731 return Register();
4732}
4733
4734Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4735 if (RetVT.SimpleTy != MVT::v8i16)
4736 return Register();
4737 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4738 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
4739 }
4740 return Register();
4741}
4742
4743Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4744 if (RetVT.SimpleTy != MVT::v2i32)
4745 return Register();
4746 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4747 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
4748 }
4749 return Register();
4750}
4751
4752Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4753 if (RetVT.SimpleTy != MVT::v4i32)
4754 return Register();
4755 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4756 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
4757 }
4758 return Register();
4759}
4760
4761Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4762 if (RetVT.SimpleTy != MVT::v2i64)
4763 return Register();
4764 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4765 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
4766 }
4767 return Register();
4768}
4769
4770Register fastEmit_ISD_STRICT_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
4771 switch (VT.SimpleTy) {
4772 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
4773 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
4774 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
4775 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
4776 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
4777 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
4778 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
4779 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0);
4780 default: return Register();
4781 }
4782}
4783
4784// FastEmit functions for ISD::STRICT_FP_TO_UINT.
4785
4786Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
4787 if ((Subtarget->hasFPRCVT())) {
4788 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSHr, RC: &AArch64::FPR32RegClass, Op0);
4789 }
4790 if ((Subtarget->hasFullFP16())) {
4791 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
4792 }
4793 return Register();
4794}
4795
4796Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
4797 if ((Subtarget->hasFPRCVT())) {
4798 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDHr, RC: &AArch64::FPR64RegClass, Op0);
4799 }
4800 if ((Subtarget->hasFullFP16())) {
4801 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
4802 }
4803 return Register();
4804}
4805
4806Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
4807switch (RetVT.SimpleTy) {
4808 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
4809 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
4810 default: return Register();
4811}
4812}
4813
4814Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
4815 if ((Subtarget->hasFPARMv8())) {
4816 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
4817 }
4818 return Register();
4819}
4820
4821Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
4822 if ((Subtarget->hasFPRCVT())) {
4823 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDSr, RC: &AArch64::FPR64RegClass, Op0);
4824 }
4825 if ((Subtarget->hasFPARMv8())) {
4826 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
4827 }
4828 return Register();
4829}
4830
4831Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
4832switch (RetVT.SimpleTy) {
4833 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
4834 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
4835 default: return Register();
4836}
4837}
4838
4839Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
4840 if ((Subtarget->hasFPRCVT())) {
4841 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSDr, RC: &AArch64::FPR32RegClass, Op0);
4842 }
4843 if ((Subtarget->hasFPARMv8())) {
4844 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
4845 }
4846 return Register();
4847}
4848
4849Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
4850 if ((Subtarget->hasFPARMv8())) {
4851 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
4852 }
4853 return Register();
4854}
4855
4856Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
4857switch (RetVT.SimpleTy) {
4858 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
4859 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
4860 default: return Register();
4861}
4862}
4863
4864Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4865 if (RetVT.SimpleTy != MVT::v4i16)
4866 return Register();
4867 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4868 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
4869 }
4870 return Register();
4871}
4872
4873Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4874 if (RetVT.SimpleTy != MVT::v8i16)
4875 return Register();
4876 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4877 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
4878 }
4879 return Register();
4880}
4881
4882Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4883 if (RetVT.SimpleTy != MVT::v2i32)
4884 return Register();
4885 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4886 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
4887 }
4888 return Register();
4889}
4890
4891Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4892 if (RetVT.SimpleTy != MVT::v4i32)
4893 return Register();
4894 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4895 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
4896 }
4897 return Register();
4898}
4899
4900Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4901 if (RetVT.SimpleTy != MVT::v2i64)
4902 return Register();
4903 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4904 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
4905 }
4906 return Register();
4907}
4908
4909Register fastEmit_ISD_STRICT_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
4910 switch (VT.SimpleTy) {
4911 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
4912 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
4913 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
4914 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
4915 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
4916 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
4917 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
4918 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0);
4919 default: return Register();
4920 }
4921}
4922
4923// FastEmit functions for ISD::STRICT_FRINT.
4924
4925Register fastEmit_ISD_STRICT_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
4926 if (RetVT.SimpleTy != MVT::f16)
4927 return Register();
4928 if ((Subtarget->hasFullFP16())) {
4929 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXHr, RC: &AArch64::FPR16RegClass, Op0);
4930 }
4931 return Register();
4932}
4933
4934Register fastEmit_ISD_STRICT_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
4935 if (RetVT.SimpleTy != MVT::f32)
4936 return Register();
4937 if ((Subtarget->hasFPARMv8())) {
4938 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXSr, RC: &AArch64::FPR32RegClass, Op0);
4939 }
4940 return Register();
4941}
4942
4943Register fastEmit_ISD_STRICT_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
4944 if (RetVT.SimpleTy != MVT::f64)
4945 return Register();
4946 if ((Subtarget->hasFPARMv8())) {
4947 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXDr, RC: &AArch64::FPR64RegClass, Op0);
4948 }
4949 return Register();
4950}
4951
4952Register fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4953 if (RetVT.SimpleTy != MVT::v4f16)
4954 return Register();
4955 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4956 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f16, RC: &AArch64::FPR64RegClass, Op0);
4957 }
4958 return Register();
4959}
4960
4961Register fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4962 if (RetVT.SimpleTy != MVT::v8f16)
4963 return Register();
4964 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4965 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv8f16, RC: &AArch64::FPR128RegClass, Op0);
4966 }
4967 return Register();
4968}
4969
4970Register fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4971 if (RetVT.SimpleTy != MVT::v2f32)
4972 return Register();
4973 if ((Subtarget->isNeonAvailable())) {
4974 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f32, RC: &AArch64::FPR64RegClass, Op0);
4975 }
4976 return Register();
4977}
4978
4979Register fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4980 if (RetVT.SimpleTy != MVT::v4f32)
4981 return Register();
4982 if ((Subtarget->isNeonAvailable())) {
4983 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f32, RC: &AArch64::FPR128RegClass, Op0);
4984 }
4985 return Register();
4986}
4987
4988Register fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4989 if (RetVT.SimpleTy != MVT::v2f64)
4990 return Register();
4991 if ((Subtarget->isNeonAvailable())) {
4992 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f64, RC: &AArch64::FPR128RegClass, Op0);
4993 }
4994 return Register();
4995}
4996
4997Register fastEmit_ISD_STRICT_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
4998 switch (VT.SimpleTy) {
4999 case MVT::f16: return fastEmit_ISD_STRICT_FRINT_MVT_f16_r(RetVT, Op0);
5000 case MVT::f32: return fastEmit_ISD_STRICT_FRINT_MVT_f32_r(RetVT, Op0);
5001 case MVT::f64: return fastEmit_ISD_STRICT_FRINT_MVT_f64_r(RetVT, Op0);
5002 case MVT::v4f16: return fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(RetVT, Op0);
5003 case MVT::v8f16: return fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(RetVT, Op0);
5004 case MVT::v2f32: return fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(RetVT, Op0);
5005 case MVT::v4f32: return fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(RetVT, Op0);
5006 case MVT::v2f64: return fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(RetVT, Op0);
5007 default: return Register();
5008 }
5009}
5010
5011// FastEmit functions for ISD::STRICT_FROUND.
5012
5013Register fastEmit_ISD_STRICT_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5014 if (RetVT.SimpleTy != MVT::f16)
5015 return Register();
5016 if ((Subtarget->hasFullFP16())) {
5017 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAHr, RC: &AArch64::FPR16RegClass, Op0);
5018 }
5019 return Register();
5020}
5021
5022Register fastEmit_ISD_STRICT_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5023 if (RetVT.SimpleTy != MVT::f32)
5024 return Register();
5025 if ((Subtarget->hasFPARMv8())) {
5026 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTASr, RC: &AArch64::FPR32RegClass, Op0);
5027 }
5028 return Register();
5029}
5030
5031Register fastEmit_ISD_STRICT_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5032 if (RetVT.SimpleTy != MVT::f64)
5033 return Register();
5034 if ((Subtarget->hasFPARMv8())) {
5035 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTADr, RC: &AArch64::FPR64RegClass, Op0);
5036 }
5037 return Register();
5038}
5039
5040Register fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
5041 if (RetVT.SimpleTy != MVT::v4f16)
5042 return Register();
5043 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5044 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f16, RC: &AArch64::FPR64RegClass, Op0);
5045 }
5046 return Register();
5047}
5048
5049Register fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
5050 if (RetVT.SimpleTy != MVT::v8f16)
5051 return Register();
5052 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5053 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv8f16, RC: &AArch64::FPR128RegClass, Op0);
5054 }
5055 return Register();
5056}
5057
5058Register fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
5059 if (RetVT.SimpleTy != MVT::v2f32)
5060 return Register();
5061 if ((Subtarget->isNeonAvailable())) {
5062 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f32, RC: &AArch64::FPR64RegClass, Op0);
5063 }
5064 return Register();
5065}
5066
5067Register fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
5068 if (RetVT.SimpleTy != MVT::v4f32)
5069 return Register();
5070 if ((Subtarget->isNeonAvailable())) {
5071 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f32, RC: &AArch64::FPR128RegClass, Op0);
5072 }
5073 return Register();
5074}
5075
5076Register fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
5077 if (RetVT.SimpleTy != MVT::v2f64)
5078 return Register();
5079 if ((Subtarget->isNeonAvailable())) {
5080 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f64, RC: &AArch64::FPR128RegClass, Op0);
5081 }
5082 return Register();
5083}
5084
5085Register fastEmit_ISD_STRICT_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
5086 switch (VT.SimpleTy) {
5087 case MVT::f16: return fastEmit_ISD_STRICT_FROUND_MVT_f16_r(RetVT, Op0);
5088 case MVT::f32: return fastEmit_ISD_STRICT_FROUND_MVT_f32_r(RetVT, Op0);
5089 case MVT::f64: return fastEmit_ISD_STRICT_FROUND_MVT_f64_r(RetVT, Op0);
5090 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(RetVT, Op0);
5091 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(RetVT, Op0);
5092 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(RetVT, Op0);
5093 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(RetVT, Op0);
5094 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(RetVT, Op0);
5095 default: return Register();
5096 }
5097}
5098
5099// FastEmit functions for ISD::STRICT_FROUNDEVEN.
5100
5101Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
5102 if (RetVT.SimpleTy != MVT::f16)
5103 return Register();
5104 if ((Subtarget->hasFullFP16())) {
5105 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNHr, RC: &AArch64::FPR16RegClass, Op0);
5106 }
5107 return Register();
5108}
5109
5110Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
5111 if (RetVT.SimpleTy != MVT::f32)
5112 return Register();
5113 if ((Subtarget->hasFPARMv8())) {
5114 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNSr, RC: &AArch64::FPR32RegClass, Op0);
5115 }
5116 return Register();
5117}
5118
5119Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
5120 if (RetVT.SimpleTy != MVT::f64)
5121 return Register();
5122 if ((Subtarget->hasFPARMv8())) {
5123 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNDr, RC: &AArch64::FPR64RegClass, Op0);
5124 }
5125 return Register();
5126}
5127
5128Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
5129 if (RetVT.SimpleTy != MVT::v4f16)
5130 return Register();
5131 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5132 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f16, RC: &AArch64::FPR64RegClass, Op0);
5133 }
5134 return Register();
5135}
5136
5137Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
5138 if (RetVT.SimpleTy != MVT::v8f16)
5139 return Register();
5140 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5141 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv8f16, RC: &AArch64::FPR128RegClass, Op0);
5142 }
5143 return Register();
5144}
5145
5146Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
5147 if (RetVT.SimpleTy != MVT::v2f32)
5148 return Register();
5149 if ((Subtarget->isNeonAvailable())) {
5150 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f32, RC: &AArch64::FPR64RegClass, Op0);
5151 }
5152 return Register();
5153}
5154
5155Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
5156 if (RetVT.SimpleTy != MVT::v4f32)
5157 return Register();
5158 if ((Subtarget->isNeonAvailable())) {
5159 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f32, RC: &AArch64::FPR128RegClass, Op0);
5160 }
5161 return Register();
5162}
5163
5164Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, Register Op0) {
5165 if (RetVT.SimpleTy != MVT::v2f64)
5166 return Register();
5167 if ((Subtarget->isNeonAvailable())) {
5168 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f64, RC: &AArch64::FPR128RegClass, Op0);
5169 }
5170 return Register();
5171}
5172
5173Register fastEmit_ISD_STRICT_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
5174 switch (VT.SimpleTy) {
5175 case MVT::f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
5176 case MVT::f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
5177 case MVT::f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
5178 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
5179 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
5180 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
5181 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
5182 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0);
5183 default: return Register();
5184 }
5185}
5186
5187// FastEmit functions for ISD::STRICT_FSQRT.
5188
5189Register fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
5190 if (RetVT.SimpleTy != MVT::f16)
5191 return Register();
5192 if ((Subtarget->hasFullFP16())) {
5193 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTHr, RC: &AArch64::FPR16RegClass, Op0);
5194 }
5195 return Register();
5196}
5197
5198Register fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
5199 if (RetVT.SimpleTy != MVT::f32)
5200 return Register();
5201 if ((Subtarget->hasFPARMv8())) {
5202 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTSr, RC: &AArch64::FPR32RegClass, Op0);
5203 }
5204 return Register();
5205}
5206
5207Register fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
5208 if (RetVT.SimpleTy != MVT::f64)
5209 return Register();
5210 if ((Subtarget->hasFPARMv8())) {
5211 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTDr, RC: &AArch64::FPR64RegClass, Op0);
5212 }
5213 return Register();
5214}
5215
5216Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(MVT RetVT, Register Op0) {
5217 if (RetVT.SimpleTy != MVT::v4f16)
5218 return Register();
5219 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5220 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f16, RC: &AArch64::FPR64RegClass, Op0);
5221 }
5222 return Register();
5223}
5224
5225Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
5226 if (RetVT.SimpleTy != MVT::v8f16)
5227 return Register();
5228 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5229 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv8f16, RC: &AArch64::FPR128RegClass, Op0);
5230 }
5231 return Register();
5232}
5233
5234Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(MVT RetVT, Register Op0) {
5235 if (RetVT.SimpleTy != MVT::v2f32)
5236 return Register();
5237 if ((Subtarget->isNeonAvailable())) {
5238 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f32, RC: &AArch64::FPR64RegClass, Op0);
5239 }
5240 return Register();
5241}
5242
5243Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
5244 if (RetVT.SimpleTy != MVT::v4f32)
5245 return Register();
5246 if ((Subtarget->isNeonAvailable())) {
5247 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f32, RC: &AArch64::FPR128RegClass, Op0);
5248 }
5249 return Register();
5250}
5251
5252Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
5253 if (RetVT.SimpleTy != MVT::v2f64)
5254 return Register();
5255 if ((Subtarget->isNeonAvailable())) {
5256 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f64, RC: &AArch64::FPR128RegClass, Op0);
5257 }
5258 return Register();
5259}
5260
5261Register fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
5262 switch (VT.SimpleTy) {
5263 case MVT::f16: return fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(RetVT, Op0);
5264 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0);
5265 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0);
5266 case MVT::v4f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(RetVT, Op0);
5267 case MVT::v8f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(RetVT, Op0);
5268 case MVT::v2f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(RetVT, Op0);
5269 case MVT::v4f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(RetVT, Op0);
5270 case MVT::v2f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(RetVT, Op0);
5271 default: return Register();
5272 }
5273}
5274
5275// FastEmit functions for ISD::STRICT_FTRUNC.
5276
5277Register fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
5278 if (RetVT.SimpleTy != MVT::f16)
5279 return Register();
5280 if ((Subtarget->hasFullFP16())) {
5281 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZHr, RC: &AArch64::FPR16RegClass, Op0);
5282 }
5283 return Register();
5284}
5285
5286Register fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
5287 if (RetVT.SimpleTy != MVT::f32)
5288 return Register();
5289 if ((Subtarget->hasFPARMv8())) {
5290 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZSr, RC: &AArch64::FPR32RegClass, Op0);
5291 }
5292 return Register();
5293}
5294
5295Register fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
5296 if (RetVT.SimpleTy != MVT::f64)
5297 return Register();
5298 if ((Subtarget->hasFPARMv8())) {
5299 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZDr, RC: &AArch64::FPR64RegClass, Op0);
5300 }
5301 return Register();
5302}
5303
5304Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
5305 if (RetVT.SimpleTy != MVT::v4f16)
5306 return Register();
5307 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5308 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f16, RC: &AArch64::FPR64RegClass, Op0);
5309 }
5310 return Register();
5311}
5312
5313Register fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
5314 if (RetVT.SimpleTy != MVT::v8f16)
5315 return Register();
5316 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5317 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv8f16, RC: &AArch64::FPR128RegClass, Op0);
5318 }
5319 return Register();
5320}
5321
5322Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
5323 if (RetVT.SimpleTy != MVT::v2f32)
5324 return Register();
5325 if ((Subtarget->isNeonAvailable())) {
5326 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f32, RC: &AArch64::FPR64RegClass, Op0);
5327 }
5328 return Register();
5329}
5330
5331Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
5332 if (RetVT.SimpleTy != MVT::v4f32)
5333 return Register();
5334 if ((Subtarget->isNeonAvailable())) {
5335 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f32, RC: &AArch64::FPR128RegClass, Op0);
5336 }
5337 return Register();
5338}
5339
5340Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(MVT RetVT, Register Op0) {
5341 if (RetVT.SimpleTy != MVT::v2f64)
5342 return Register();
5343 if ((Subtarget->isNeonAvailable())) {
5344 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f64, RC: &AArch64::FPR128RegClass, Op0);
5345 }
5346 return Register();
5347}
5348
5349Register fastEmit_ISD_STRICT_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
5350 switch (VT.SimpleTy) {
5351 case MVT::f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(RetVT, Op0);
5352 case MVT::f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(RetVT, Op0);
5353 case MVT::f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(RetVT, Op0);
5354 case MVT::v4f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(RetVT, Op0);
5355 case MVT::v8f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(RetVT, Op0);
5356 case MVT::v2f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(RetVT, Op0);
5357 case MVT::v4f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(RetVT, Op0);
5358 case MVT::v2f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(RetVT, Op0);
5359 default: return Register();
5360 }
5361}
5362
5363// FastEmit functions for ISD::STRICT_LLROUND.
5364
5365Register fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5366 if (RetVT.SimpleTy != MVT::i64)
5367 return Register();
5368 if ((Subtarget->hasFullFP16())) {
5369 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
5370 }
5371 return Register();
5372}
5373
5374Register fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5375 if (RetVT.SimpleTy != MVT::i64)
5376 return Register();
5377 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
5378}
5379
5380Register fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5381 if (RetVT.SimpleTy != MVT::i64)
5382 return Register();
5383 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
5384}
5385
5386Register fastEmit_ISD_STRICT_LLROUND_r(MVT VT, MVT RetVT, Register Op0) {
5387 switch (VT.SimpleTy) {
5388 case MVT::f16: return fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(RetVT, Op0);
5389 case MVT::f32: return fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(RetVT, Op0);
5390 case MVT::f64: return fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(RetVT, Op0);
5391 default: return Register();
5392 }
5393}
5394
5395// FastEmit functions for ISD::STRICT_LROUND.
5396
5397Register fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(Register Op0) {
5398 if ((Subtarget->hasFullFP16())) {
5399 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWHr, RC: &AArch64::GPR32RegClass, Op0);
5400 }
5401 return Register();
5402}
5403
5404Register fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(Register Op0) {
5405 if ((Subtarget->hasFullFP16())) {
5406 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
5407 }
5408 return Register();
5409}
5410
5411Register fastEmit_ISD_STRICT_LROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5412switch (RetVT.SimpleTy) {
5413 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(Op0);
5414 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(Op0);
5415 default: return Register();
5416}
5417}
5418
5419Register fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(Register Op0) {
5420 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWSr, RC: &AArch64::GPR32RegClass, Op0);
5421}
5422
5423Register fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(Register Op0) {
5424 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
5425}
5426
5427Register fastEmit_ISD_STRICT_LROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5428switch (RetVT.SimpleTy) {
5429 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(Op0);
5430 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(Op0);
5431 default: return Register();
5432}
5433}
5434
5435Register fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(Register Op0) {
5436 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWDr, RC: &AArch64::GPR32RegClass, Op0);
5437}
5438
5439Register fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(Register Op0) {
5440 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
5441}
5442
5443Register fastEmit_ISD_STRICT_LROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5444switch (RetVT.SimpleTy) {
5445 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(Op0);
5446 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(Op0);
5447 default: return Register();
5448}
5449}
5450
5451Register fastEmit_ISD_STRICT_LROUND_r(MVT VT, MVT RetVT, Register Op0) {
5452 switch (VT.SimpleTy) {
5453 case MVT::f16: return fastEmit_ISD_STRICT_LROUND_MVT_f16_r(RetVT, Op0);
5454 case MVT::f32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_r(RetVT, Op0);
5455 case MVT::f64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_r(RetVT, Op0);
5456 default: return Register();
5457 }
5458}
5459
5460// FastEmit functions for ISD::STRICT_SINT_TO_FP.
5461
5462Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5463 if ((Subtarget->hasFPRCVT())) {
5464 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5465 }
5466 if ((Subtarget->hasFullFP16())) {
5467 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5468 }
5469 return Register();
5470}
5471
5472Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5473 if ((Subtarget->hasFPARMv8())) {
5474 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5475 }
5476 return Register();
5477}
5478
5479Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5480 if ((Subtarget->hasFPRCVT())) {
5481 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5482 }
5483 if ((Subtarget->hasFPARMv8())) {
5484 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5485 }
5486 return Register();
5487}
5488
5489Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5490switch (RetVT.SimpleTy) {
5491 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5492 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5493 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5494 default: return Register();
5495}
5496}
5497
5498Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5499 if ((Subtarget->hasFPRCVT())) {
5500 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5501 }
5502 if ((Subtarget->hasFullFP16())) {
5503 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5504 }
5505 return Register();
5506}
5507
5508Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5509 if ((Subtarget->hasFPRCVT())) {
5510 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5511 }
5512 if ((Subtarget->hasFPARMv8())) {
5513 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5514 }
5515 return Register();
5516}
5517
5518Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5519 if ((Subtarget->hasFPARMv8())) {
5520 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5521 }
5522 return Register();
5523}
5524
5525Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5526switch (RetVT.SimpleTy) {
5527 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5528 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5529 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5530 default: return Register();
5531}
5532}
5533
5534Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5535 if (RetVT.SimpleTy != MVT::v4f16)
5536 return Register();
5537 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5538 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5539 }
5540 return Register();
5541}
5542
5543Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5544 if (RetVT.SimpleTy != MVT::v8f16)
5545 return Register();
5546 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5547 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5548 }
5549 return Register();
5550}
5551
5552Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5553 if (RetVT.SimpleTy != MVT::v2f32)
5554 return Register();
5555 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5556 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5557 }
5558 return Register();
5559}
5560
5561Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5562 if (RetVT.SimpleTy != MVT::v4f32)
5563 return Register();
5564 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5565 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5566 }
5567 return Register();
5568}
5569
5570Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5571 if (RetVT.SimpleTy != MVT::v2f64)
5572 return Register();
5573 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5574 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5575 }
5576 return Register();
5577}
5578
5579Register fastEmit_ISD_STRICT_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
5580 switch (VT.SimpleTy) {
5581 case MVT::i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
5582 case MVT::i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
5583 case MVT::v4i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
5584 case MVT::v8i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
5585 case MVT::v2i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
5586 case MVT::v4i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
5587 case MVT::v2i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
5588 default: return Register();
5589 }
5590}
5591
5592// FastEmit functions for ISD::STRICT_UINT_TO_FP.
5593
5594Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5595 if ((Subtarget->hasFPRCVT())) {
5596 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5597 }
5598 if ((Subtarget->hasFullFP16())) {
5599 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5600 }
5601 return Register();
5602}
5603
5604Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5605 if ((Subtarget->hasFPARMv8())) {
5606 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5607 }
5608 return Register();
5609}
5610
5611Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5612 if ((Subtarget->hasFPRCVT())) {
5613 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5614 }
5615 if ((Subtarget->hasFPARMv8())) {
5616 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5617 }
5618 return Register();
5619}
5620
5621Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5622switch (RetVT.SimpleTy) {
5623 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5624 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5625 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5626 default: return Register();
5627}
5628}
5629
5630Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5631 if ((Subtarget->hasFPRCVT())) {
5632 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5633 }
5634 if ((Subtarget->hasFullFP16())) {
5635 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5636 }
5637 return Register();
5638}
5639
5640Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5641 if ((Subtarget->hasFPRCVT())) {
5642 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5643 }
5644 if ((Subtarget->hasFPARMv8())) {
5645 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5646 }
5647 return Register();
5648}
5649
5650Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5651 if ((Subtarget->hasFPARMv8())) {
5652 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5653 }
5654 return Register();
5655}
5656
5657Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5658switch (RetVT.SimpleTy) {
5659 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5660 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5661 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5662 default: return Register();
5663}
5664}
5665
5666Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5667 if (RetVT.SimpleTy != MVT::v4f16)
5668 return Register();
5669 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5670 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5671 }
5672 return Register();
5673}
5674
5675Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5676 if (RetVT.SimpleTy != MVT::v8f16)
5677 return Register();
5678 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5679 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5680 }
5681 return Register();
5682}
5683
5684Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5685 if (RetVT.SimpleTy != MVT::v2f32)
5686 return Register();
5687 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5688 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5689 }
5690 return Register();
5691}
5692
5693Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5694 if (RetVT.SimpleTy != MVT::v4f32)
5695 return Register();
5696 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5697 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5698 }
5699 return Register();
5700}
5701
5702Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5703 if (RetVT.SimpleTy != MVT::v2f64)
5704 return Register();
5705 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5706 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5707 }
5708 return Register();
5709}
5710
5711Register fastEmit_ISD_STRICT_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
5712 switch (VT.SimpleTy) {
5713 case MVT::i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(RetVT, Op0);
5714 case MVT::i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(RetVT, Op0);
5715 case MVT::v4i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
5716 case MVT::v8i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
5717 case MVT::v2i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
5718 case MVT::v4i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
5719 case MVT::v2i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
5720 default: return Register();
5721 }
5722}
5723
5724// FastEmit functions for ISD::TRUNCATE.
5725
5726Register fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, Register Op0) {
5727 if (RetVT.SimpleTy != MVT::i32)
5728 return Register();
5729 return fastEmitInst_extractsubreg(RetVT, Op0, Idx: AArch64::sub_32);
5730}
5731
5732Register fastEmit_ISD_TRUNCATE_MVT_v8i16_r(MVT RetVT, Register Op0) {
5733 if (RetVT.SimpleTy != MVT::v8i8)
5734 return Register();
5735 if ((Subtarget->isNeonAvailable())) {
5736 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5737 }
5738 return Register();
5739}
5740
5741Register fastEmit_ISD_TRUNCATE_MVT_v4i32_r(MVT RetVT, Register Op0) {
5742 if (RetVT.SimpleTy != MVT::v4i16)
5743 return Register();
5744 if ((Subtarget->isNeonAvailable())) {
5745 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5746 }
5747 return Register();
5748}
5749
5750Register fastEmit_ISD_TRUNCATE_MVT_v2i64_r(MVT RetVT, Register Op0) {
5751 if (RetVT.SimpleTy != MVT::v2i32)
5752 return Register();
5753 if ((Subtarget->isNeonAvailable())) {
5754 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5755 }
5756 return Register();
5757}
5758
5759Register fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, Register Op0) {
5760 switch (VT.SimpleTy) {
5761 case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0);
5762 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i16_r(RetVT, Op0);
5763 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_MVT_v4i32_r(RetVT, Op0);
5764 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_MVT_v2i64_r(RetVT, Op0);
5765 default: return Register();
5766 }
5767}
5768
5769// FastEmit functions for ISD::TRUNCATE_SSAT_S.
5770
5771Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v8i16_r(MVT RetVT, Register Op0) {
5772 if (RetVT.SimpleTy != MVT::v8i8)
5773 return Register();
5774 if ((Subtarget->isNeonAvailable())) {
5775 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5776 }
5777 return Register();
5778}
5779
5780Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v4i32_r(MVT RetVT, Register Op0) {
5781 if (RetVT.SimpleTy != MVT::v4i16)
5782 return Register();
5783 if ((Subtarget->isNeonAvailable())) {
5784 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5785 }
5786 return Register();
5787}
5788
5789Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v2i64_r(MVT RetVT, Register Op0) {
5790 if (RetVT.SimpleTy != MVT::v2i32)
5791 return Register();
5792 if ((Subtarget->isNeonAvailable())) {
5793 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5794 }
5795 return Register();
5796}
5797
5798Register fastEmit_ISD_TRUNCATE_SSAT_S_r(MVT VT, MVT RetVT, Register Op0) {
5799 switch (VT.SimpleTy) {
5800 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v8i16_r(RetVT, Op0);
5801 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v4i32_r(RetVT, Op0);
5802 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v2i64_r(RetVT, Op0);
5803 default: return Register();
5804 }
5805}
5806
5807// FastEmit functions for ISD::TRUNCATE_SSAT_U.
5808
5809Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v8i16_r(MVT RetVT, Register Op0) {
5810 if (RetVT.SimpleTy != MVT::v8i8)
5811 return Register();
5812 if ((Subtarget->isNeonAvailable())) {
5813 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5814 }
5815 return Register();
5816}
5817
5818Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v4i32_r(MVT RetVT, Register Op0) {
5819 if (RetVT.SimpleTy != MVT::v4i16)
5820 return Register();
5821 if ((Subtarget->isNeonAvailable())) {
5822 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5823 }
5824 return Register();
5825}
5826
5827Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v2i64_r(MVT RetVT, Register Op0) {
5828 if (RetVT.SimpleTy != MVT::v2i32)
5829 return Register();
5830 if ((Subtarget->isNeonAvailable())) {
5831 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5832 }
5833 return Register();
5834}
5835
5836Register fastEmit_ISD_TRUNCATE_SSAT_U_r(MVT VT, MVT RetVT, Register Op0) {
5837 switch (VT.SimpleTy) {
5838 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v8i16_r(RetVT, Op0);
5839 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v4i32_r(RetVT, Op0);
5840 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v2i64_r(RetVT, Op0);
5841 default: return Register();
5842 }
5843}
5844
5845// FastEmit functions for ISD::TRUNCATE_USAT_U.
5846
5847Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v8i16_r(MVT RetVT, Register Op0) {
5848 if (RetVT.SimpleTy != MVT::v8i8)
5849 return Register();
5850 if ((Subtarget->isNeonAvailable())) {
5851 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5852 }
5853 return Register();
5854}
5855
5856Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v4i32_r(MVT RetVT, Register Op0) {
5857 if (RetVT.SimpleTy != MVT::v4i16)
5858 return Register();
5859 if ((Subtarget->isNeonAvailable())) {
5860 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5861 }
5862 return Register();
5863}
5864
5865Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v2i64_r(MVT RetVT, Register Op0) {
5866 if (RetVT.SimpleTy != MVT::v2i32)
5867 return Register();
5868 if ((Subtarget->isNeonAvailable())) {
5869 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5870 }
5871 return Register();
5872}
5873
5874Register fastEmit_ISD_TRUNCATE_USAT_U_r(MVT VT, MVT RetVT, Register Op0) {
5875 switch (VT.SimpleTy) {
5876 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v8i16_r(RetVT, Op0);
5877 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v4i32_r(RetVT, Op0);
5878 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v2i64_r(RetVT, Op0);
5879 default: return Register();
5880 }
5881}
5882
5883// FastEmit functions for ISD::UINT_TO_FP.
5884
5885Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5886 if ((Subtarget->hasFPRCVT())) {
5887 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5888 }
5889 if ((Subtarget->hasFullFP16())) {
5890 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5891 }
5892 return Register();
5893}
5894
5895Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5896 if ((Subtarget->hasFPARMv8())) {
5897 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5898 }
5899 return Register();
5900}
5901
5902Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5903 if ((Subtarget->hasFPRCVT())) {
5904 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5905 }
5906 if ((Subtarget->hasFPARMv8())) {
5907 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5908 }
5909 return Register();
5910}
5911
5912Register fastEmit_ISD_UINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5913switch (RetVT.SimpleTy) {
5914 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5915 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5916 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5917 default: return Register();
5918}
5919}
5920
5921Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5922 if ((Subtarget->hasFPRCVT())) {
5923 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5924 }
5925 if ((Subtarget->hasFullFP16())) {
5926 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5927 }
5928 return Register();
5929}
5930
5931Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5932 if ((Subtarget->hasFPRCVT())) {
5933 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5934 }
5935 if ((Subtarget->hasFPARMv8())) {
5936 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5937 }
5938 return Register();
5939}
5940
5941Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5942 if ((Subtarget->hasFPARMv8())) {
5943 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5944 }
5945 return Register();
5946}
5947
5948Register fastEmit_ISD_UINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5949switch (RetVT.SimpleTy) {
5950 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5951 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5952 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5953 default: return Register();
5954}
5955}
5956
5957Register fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5958 if (RetVT.SimpleTy != MVT::v4f16)
5959 return Register();
5960 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5961 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5962 }
5963 return Register();
5964}
5965
5966Register fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5967 if (RetVT.SimpleTy != MVT::v8f16)
5968 return Register();
5969 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5970 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5971 }
5972 return Register();
5973}
5974
5975Register fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5976 if (RetVT.SimpleTy != MVT::v2f32)
5977 return Register();
5978 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5979 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5980 }
5981 return Register();
5982}
5983
5984Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5985 if (RetVT.SimpleTy != MVT::v4f32)
5986 return Register();
5987 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5988 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5989 }
5990 return Register();
5991}
5992
5993Register fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5994 if (RetVT.SimpleTy != MVT::v2f64)
5995 return Register();
5996 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5997 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5998 }
5999 return Register();
6000}
6001
6002Register fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
6003 switch (VT.SimpleTy) {
6004 case MVT::i32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_r(RetVT, Op0);
6005 case MVT::i64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_r(RetVT, Op0);
6006 case MVT::v4i16: return fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
6007 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
6008 case MVT::v2i32: return fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
6009 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
6010 case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
6011 default: return Register();
6012 }
6013}
6014
6015// FastEmit functions for ISD::VECREDUCE_ADD.
6016
6017Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i8_r(MVT RetVT, Register Op0) {
6018 if (RetVT.SimpleTy != MVT::i8)
6019 return Register();
6020 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6021}
6022
6023Register fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(MVT RetVT, Register Op0) {
6024 if (RetVT.SimpleTy != MVT::i8)
6025 return Register();
6026 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6027}
6028
6029Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i16_r(MVT RetVT, Register Op0) {
6030 if (RetVT.SimpleTy != MVT::i16)
6031 return Register();
6032 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6033}
6034
6035Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(MVT RetVT, Register Op0) {
6036 if (RetVT.SimpleTy != MVT::i16)
6037 return Register();
6038 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6039}
6040
6041Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(MVT RetVT, Register Op0) {
6042 if (RetVT.SimpleTy != MVT::i32)
6043 return Register();
6044 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6045}
6046
6047Register fastEmit_ISD_VECREDUCE_ADD_MVT_v2i64_r(MVT RetVT, Register Op0) {
6048 if (RetVT.SimpleTy != MVT::i64)
6049 return Register();
6050 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6051}
6052
6053Register fastEmit_ISD_VECREDUCE_ADD_r(MVT VT, MVT RetVT, Register Op0) {
6054 switch (VT.SimpleTy) {
6055 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i8_r(RetVT, Op0);
6056 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(RetVT, Op0);
6057 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i16_r(RetVT, Op0);
6058 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(RetVT, Op0);
6059 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(RetVT, Op0);
6060 case MVT::v2i64: return fastEmit_ISD_VECREDUCE_ADD_MVT_v2i64_r(RetVT, Op0);
6061 default: return Register();
6062 }
6063}
6064
6065// FastEmit functions for ISD::VECREDUCE_FADD.
6066
6067Register fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(MVT RetVT, Register Op0) {
6068 if (RetVT.SimpleTy != MVT::f32)
6069 return Register();
6070 return fastEmitInst_r(MachineInstOpcode: AArch64::FADDPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6071}
6072
6073Register fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(MVT RetVT, Register Op0) {
6074 if (RetVT.SimpleTy != MVT::f64)
6075 return Register();
6076 return fastEmitInst_r(MachineInstOpcode: AArch64::FADDPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6077}
6078
6079Register fastEmit_ISD_VECREDUCE_FADD_r(MVT VT, MVT RetVT, Register Op0) {
6080 switch (VT.SimpleTy) {
6081 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(RetVT, Op0);
6082 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(RetVT, Op0);
6083 default: return Register();
6084 }
6085}
6086
6087// FastEmit functions for ISD::VECREDUCE_FMAX.
6088
6089Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f16_r(MVT RetVT, Register Op0) {
6090 if (RetVT.SimpleTy != MVT::f16)
6091 return Register();
6092 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6093 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6094 }
6095 return Register();
6096}
6097
6098Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v8f16_r(MVT RetVT, Register Op0) {
6099 if (RetVT.SimpleTy != MVT::f16)
6100 return Register();
6101 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6102 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6103 }
6104 return Register();
6105}
6106
6107Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f32_r(MVT RetVT, Register Op0) {
6108 if (RetVT.SimpleTy != MVT::f32)
6109 return Register();
6110 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6111}
6112
6113Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f32_r(MVT RetVT, Register Op0) {
6114 if (RetVT.SimpleTy != MVT::f32)
6115 return Register();
6116 if ((Subtarget->isNeonAvailable())) {
6117 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6118 }
6119 return Register();
6120}
6121
6122Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f64_r(MVT RetVT, Register Op0) {
6123 if (RetVT.SimpleTy != MVT::f64)
6124 return Register();
6125 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6126}
6127
6128Register fastEmit_ISD_VECREDUCE_FMAX_r(MVT VT, MVT RetVT, Register Op0) {
6129 switch (VT.SimpleTy) {
6130 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f16_r(RetVT, Op0);
6131 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v8f16_r(RetVT, Op0);
6132 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f32_r(RetVT, Op0);
6133 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f32_r(RetVT, Op0);
6134 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f64_r(RetVT, Op0);
6135 default: return Register();
6136 }
6137}
6138
6139// FastEmit functions for ISD::VECREDUCE_FMAXIMUM.
6140
6141Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f16_r(MVT RetVT, Register Op0) {
6142 if (RetVT.SimpleTy != MVT::f16)
6143 return Register();
6144 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6145 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6146 }
6147 return Register();
6148}
6149
6150Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v8f16_r(MVT RetVT, Register Op0) {
6151 if (RetVT.SimpleTy != MVT::f16)
6152 return Register();
6153 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6154 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6155 }
6156 return Register();
6157}
6158
6159Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f32_r(MVT RetVT, Register Op0) {
6160 if (RetVT.SimpleTy != MVT::f32)
6161 return Register();
6162 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6163}
6164
6165Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f32_r(MVT RetVT, Register Op0) {
6166 if (RetVT.SimpleTy != MVT::f32)
6167 return Register();
6168 if ((Subtarget->isNeonAvailable())) {
6169 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6170 }
6171 return Register();
6172}
6173
6174Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f64_r(MVT RetVT, Register Op0) {
6175 if (RetVT.SimpleTy != MVT::f64)
6176 return Register();
6177 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6178}
6179
6180Register fastEmit_ISD_VECREDUCE_FMAXIMUM_r(MVT VT, MVT RetVT, Register Op0) {
6181 switch (VT.SimpleTy) {
6182 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f16_r(RetVT, Op0);
6183 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v8f16_r(RetVT, Op0);
6184 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f32_r(RetVT, Op0);
6185 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f32_r(RetVT, Op0);
6186 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f64_r(RetVT, Op0);
6187 default: return Register();
6188 }
6189}
6190
6191// FastEmit functions for ISD::VECREDUCE_FMIN.
6192
6193Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f16_r(MVT RetVT, Register Op0) {
6194 if (RetVT.SimpleTy != MVT::f16)
6195 return Register();
6196 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6197 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6198 }
6199 return Register();
6200}
6201
6202Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v8f16_r(MVT RetVT, Register Op0) {
6203 if (RetVT.SimpleTy != MVT::f16)
6204 return Register();
6205 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6206 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6207 }
6208 return Register();
6209}
6210
6211Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f32_r(MVT RetVT, Register Op0) {
6212 if (RetVT.SimpleTy != MVT::f32)
6213 return Register();
6214 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6215}
6216
6217Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f32_r(MVT RetVT, Register Op0) {
6218 if (RetVT.SimpleTy != MVT::f32)
6219 return Register();
6220 if ((Subtarget->isNeonAvailable())) {
6221 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6222 }
6223 return Register();
6224}
6225
6226Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f64_r(MVT RetVT, Register Op0) {
6227 if (RetVT.SimpleTy != MVT::f64)
6228 return Register();
6229 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6230}
6231
6232Register fastEmit_ISD_VECREDUCE_FMIN_r(MVT VT, MVT RetVT, Register Op0) {
6233 switch (VT.SimpleTy) {
6234 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f16_r(RetVT, Op0);
6235 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v8f16_r(RetVT, Op0);
6236 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f32_r(RetVT, Op0);
6237 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f32_r(RetVT, Op0);
6238 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f64_r(RetVT, Op0);
6239 default: return Register();
6240 }
6241}
6242
6243// FastEmit functions for ISD::VECREDUCE_FMINIMUM.
6244
6245Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f16_r(MVT RetVT, Register Op0) {
6246 if (RetVT.SimpleTy != MVT::f16)
6247 return Register();
6248 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6249 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6250 }
6251 return Register();
6252}
6253
6254Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v8f16_r(MVT RetVT, Register Op0) {
6255 if (RetVT.SimpleTy != MVT::f16)
6256 return Register();
6257 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6258 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6259 }
6260 return Register();
6261}
6262
6263Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f32_r(MVT RetVT, Register Op0) {
6264 if (RetVT.SimpleTy != MVT::f32)
6265 return Register();
6266 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6267}
6268
6269Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f32_r(MVT RetVT, Register Op0) {
6270 if (RetVT.SimpleTy != MVT::f32)
6271 return Register();
6272 if ((Subtarget->isNeonAvailable())) {
6273 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6274 }
6275 return Register();
6276}
6277
6278Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f64_r(MVT RetVT, Register Op0) {
6279 if (RetVT.SimpleTy != MVT::f64)
6280 return Register();
6281 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6282}
6283
6284Register fastEmit_ISD_VECREDUCE_FMINIMUM_r(MVT VT, MVT RetVT, Register Op0) {
6285 switch (VT.SimpleTy) {
6286 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f16_r(RetVT, Op0);
6287 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v8f16_r(RetVT, Op0);
6288 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f32_r(RetVT, Op0);
6289 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f32_r(RetVT, Op0);
6290 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f64_r(RetVT, Op0);
6291 default: return Register();
6292 }
6293}
6294
6295// FastEmit functions for ISD::VECREDUCE_SMAX.
6296
6297Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i8_r(MVT RetVT, Register Op0) {
6298 if (RetVT.SimpleTy != MVT::i8)
6299 return Register();
6300 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6301}
6302
6303Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v16i8_r(MVT RetVT, Register Op0) {
6304 if (RetVT.SimpleTy != MVT::i8)
6305 return Register();
6306 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6307}
6308
6309Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i16_r(MVT RetVT, Register Op0) {
6310 if (RetVT.SimpleTy != MVT::i16)
6311 return Register();
6312 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6313}
6314
6315Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i16_r(MVT RetVT, Register Op0) {
6316 if (RetVT.SimpleTy != MVT::i16)
6317 return Register();
6318 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6319}
6320
6321Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i32_r(MVT RetVT, Register Op0) {
6322 if (RetVT.SimpleTy != MVT::i32)
6323 return Register();
6324 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6325}
6326
6327Register fastEmit_ISD_VECREDUCE_SMAX_r(MVT VT, MVT RetVT, Register Op0) {
6328 switch (VT.SimpleTy) {
6329 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i8_r(RetVT, Op0);
6330 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v16i8_r(RetVT, Op0);
6331 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i16_r(RetVT, Op0);
6332 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i16_r(RetVT, Op0);
6333 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i32_r(RetVT, Op0);
6334 default: return Register();
6335 }
6336}
6337
6338// FastEmit functions for ISD::VECREDUCE_SMIN.
6339
6340Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i8_r(MVT RetVT, Register Op0) {
6341 if (RetVT.SimpleTy != MVT::i8)
6342 return Register();
6343 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6344}
6345
6346Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v16i8_r(MVT RetVT, Register Op0) {
6347 if (RetVT.SimpleTy != MVT::i8)
6348 return Register();
6349 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6350}
6351
6352Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i16_r(MVT RetVT, Register Op0) {
6353 if (RetVT.SimpleTy != MVT::i16)
6354 return Register();
6355 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6356}
6357
6358Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i16_r(MVT RetVT, Register Op0) {
6359 if (RetVT.SimpleTy != MVT::i16)
6360 return Register();
6361 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6362}
6363
6364Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i32_r(MVT RetVT, Register Op0) {
6365 if (RetVT.SimpleTy != MVT::i32)
6366 return Register();
6367 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6368}
6369
6370Register fastEmit_ISD_VECREDUCE_SMIN_r(MVT VT, MVT RetVT, Register Op0) {
6371 switch (VT.SimpleTy) {
6372 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i8_r(RetVT, Op0);
6373 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v16i8_r(RetVT, Op0);
6374 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i16_r(RetVT, Op0);
6375 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i16_r(RetVT, Op0);
6376 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i32_r(RetVT, Op0);
6377 default: return Register();
6378 }
6379}
6380
6381// FastEmit functions for ISD::VECREDUCE_UMAX.
6382
6383Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i8_r(MVT RetVT, Register Op0) {
6384 if (RetVT.SimpleTy != MVT::i8)
6385 return Register();
6386 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6387}
6388
6389Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v16i8_r(MVT RetVT, Register Op0) {
6390 if (RetVT.SimpleTy != MVT::i8)
6391 return Register();
6392 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6393}
6394
6395Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i16_r(MVT RetVT, Register Op0) {
6396 if (RetVT.SimpleTy != MVT::i16)
6397 return Register();
6398 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6399}
6400
6401Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i16_r(MVT RetVT, Register Op0) {
6402 if (RetVT.SimpleTy != MVT::i16)
6403 return Register();
6404 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6405}
6406
6407Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i32_r(MVT RetVT, Register Op0) {
6408 if (RetVT.SimpleTy != MVT::i32)
6409 return Register();
6410 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6411}
6412
6413Register fastEmit_ISD_VECREDUCE_UMAX_r(MVT VT, MVT RetVT, Register Op0) {
6414 switch (VT.SimpleTy) {
6415 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i8_r(RetVT, Op0);
6416 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v16i8_r(RetVT, Op0);
6417 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i16_r(RetVT, Op0);
6418 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i16_r(RetVT, Op0);
6419 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i32_r(RetVT, Op0);
6420 default: return Register();
6421 }
6422}
6423
6424// FastEmit functions for ISD::VECREDUCE_UMIN.
6425
6426Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i8_r(MVT RetVT, Register Op0) {
6427 if (RetVT.SimpleTy != MVT::i8)
6428 return Register();
6429 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6430}
6431
6432Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v16i8_r(MVT RetVT, Register Op0) {
6433 if (RetVT.SimpleTy != MVT::i8)
6434 return Register();
6435 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6436}
6437
6438Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i16_r(MVT RetVT, Register Op0) {
6439 if (RetVT.SimpleTy != MVT::i16)
6440 return Register();
6441 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6442}
6443
6444Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i16_r(MVT RetVT, Register Op0) {
6445 if (RetVT.SimpleTy != MVT::i16)
6446 return Register();
6447 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6448}
6449
6450Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i32_r(MVT RetVT, Register Op0) {
6451 if (RetVT.SimpleTy != MVT::i32)
6452 return Register();
6453 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6454}
6455
6456Register fastEmit_ISD_VECREDUCE_UMIN_r(MVT VT, MVT RetVT, Register Op0) {
6457 switch (VT.SimpleTy) {
6458 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i8_r(RetVT, Op0);
6459 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v16i8_r(RetVT, Op0);
6460 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i16_r(RetVT, Op0);
6461 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i16_r(RetVT, Op0);
6462 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i32_r(RetVT, Op0);
6463 default: return Register();
6464 }
6465}
6466
6467// FastEmit functions for ISD::VECTOR_REVERSE.
6468
6469Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(MVT RetVT, Register Op0) {
6470 if (RetVT.SimpleTy != MVT::nxv2i1)
6471 return Register();
6472 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6473 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_D, RC: &AArch64::PPRRegClass, Op0);
6474 }
6475 return Register();
6476}
6477
6478Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(MVT RetVT, Register Op0) {
6479 if (RetVT.SimpleTy != MVT::nxv4i1)
6480 return Register();
6481 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6482 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_S, RC: &AArch64::PPRRegClass, Op0);
6483 }
6484 return Register();
6485}
6486
6487Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(MVT RetVT, Register Op0) {
6488 if (RetVT.SimpleTy != MVT::nxv8i1)
6489 return Register();
6490 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6491 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_H, RC: &AArch64::PPRRegClass, Op0);
6492 }
6493 return Register();
6494}
6495
6496Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(MVT RetVT, Register Op0) {
6497 if (RetVT.SimpleTy != MVT::nxv16i1)
6498 return Register();
6499 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6500 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_B, RC: &AArch64::PPRRegClass, Op0);
6501 }
6502 return Register();
6503}
6504
6505Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
6506 if (RetVT.SimpleTy != MVT::nxv16i8)
6507 return Register();
6508 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6509 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_B, RC: &AArch64::ZPRRegClass, Op0);
6510 }
6511 return Register();
6512}
6513
6514Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
6515 if (RetVT.SimpleTy != MVT::nxv8i16)
6516 return Register();
6517 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6518 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6519 }
6520 return Register();
6521}
6522
6523Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
6524 if (RetVT.SimpleTy != MVT::nxv4i32)
6525 return Register();
6526 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6527 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6528 }
6529 return Register();
6530}
6531
6532Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(MVT RetVT, Register Op0) {
6533 if (RetVT.SimpleTy != MVT::nxv2i64)
6534 return Register();
6535 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6536 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6537 }
6538 return Register();
6539}
6540
6541Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(MVT RetVT, Register Op0) {
6542 if (RetVT.SimpleTy != MVT::nxv2f16)
6543 return Register();
6544 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6545 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6546 }
6547 return Register();
6548}
6549
6550Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(MVT RetVT, Register Op0) {
6551 if (RetVT.SimpleTy != MVT::nxv4f16)
6552 return Register();
6553 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6554 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6555 }
6556 return Register();
6557}
6558
6559Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
6560 if (RetVT.SimpleTy != MVT::nxv8f16)
6561 return Register();
6562 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6563 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6564 }
6565 return Register();
6566}
6567
6568Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(MVT RetVT, Register Op0) {
6569 if (RetVT.SimpleTy != MVT::nxv2bf16)
6570 return Register();
6571 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6572 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6573 }
6574 return Register();
6575}
6576
6577Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(MVT RetVT, Register Op0) {
6578 if (RetVT.SimpleTy != MVT::nxv4bf16)
6579 return Register();
6580 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6581 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6582 }
6583 return Register();
6584}
6585
6586Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(MVT RetVT, Register Op0) {
6587 if (RetVT.SimpleTy != MVT::nxv8bf16)
6588 return Register();
6589 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6590 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6591 }
6592 return Register();
6593}
6594
6595Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(MVT RetVT, Register Op0) {
6596 if (RetVT.SimpleTy != MVT::nxv2f32)
6597 return Register();
6598 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6599 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6600 }
6601 return Register();
6602}
6603
6604Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
6605 if (RetVT.SimpleTy != MVT::nxv4f32)
6606 return Register();
6607 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6608 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6609 }
6610 return Register();
6611}
6612
6613Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
6614 if (RetVT.SimpleTy != MVT::nxv2f64)
6615 return Register();
6616 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6617 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6618 }
6619 return Register();
6620}
6621
6622Register fastEmit_ISD_VECTOR_REVERSE_r(MVT VT, MVT RetVT, Register Op0) {
6623 switch (VT.SimpleTy) {
6624 case MVT::nxv2i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(RetVT, Op0);
6625 case MVT::nxv4i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(RetVT, Op0);
6626 case MVT::nxv8i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(RetVT, Op0);
6627 case MVT::nxv16i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(RetVT, Op0);
6628 case MVT::nxv16i8: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(RetVT, Op0);
6629 case MVT::nxv8i16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(RetVT, Op0);
6630 case MVT::nxv4i32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(RetVT, Op0);
6631 case MVT::nxv2i64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(RetVT, Op0);
6632 case MVT::nxv2f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(RetVT, Op0);
6633 case MVT::nxv4f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(RetVT, Op0);
6634 case MVT::nxv8f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(RetVT, Op0);
6635 case MVT::nxv2bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(RetVT, Op0);
6636 case MVT::nxv4bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(RetVT, Op0);
6637 case MVT::nxv8bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(RetVT, Op0);
6638 case MVT::nxv2f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(RetVT, Op0);
6639 case MVT::nxv4f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(RetVT, Op0);
6640 case MVT::nxv2f64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(RetVT, Op0);
6641 default: return Register();
6642 }
6643}
6644
6645// Top-level FastEmit function.
6646
6647Register fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, Register Op0) override {
6648 switch (Opcode) {
6649 case AArch64ISD::CALL: return fastEmit_AArch64ISD_CALL_r(VT, RetVT, Op0);
6650 case AArch64ISD::COALESCER_BARRIER: return fastEmit_AArch64ISD_COALESCER_BARRIER_r(VT, RetVT, Op0);
6651 case AArch64ISD::DUP: return fastEmit_AArch64ISD_DUP_r(VT, RetVT, Op0);
6652 case AArch64ISD::FCVTXN: return fastEmit_AArch64ISD_FCVTXN_r(VT, RetVT, Op0);
6653 case AArch64ISD::FRECPE: return fastEmit_AArch64ISD_FRECPE_r(VT, RetVT, Op0);
6654 case AArch64ISD::FRSQRTE: return fastEmit_AArch64ISD_FRSQRTE_r(VT, RetVT, Op0);
6655 case AArch64ISD::PROBED_ALLOCA: return fastEmit_AArch64ISD_PROBED_ALLOCA_r(VT, RetVT, Op0);
6656 case AArch64ISD::REV16: return fastEmit_AArch64ISD_REV16_r(VT, RetVT, Op0);
6657 case AArch64ISD::REV32: return fastEmit_AArch64ISD_REV32_r(VT, RetVT, Op0);
6658 case AArch64ISD::REV64: return fastEmit_AArch64ISD_REV64_r(VT, RetVT, Op0);
6659 case AArch64ISD::SADDLP: return fastEmit_AArch64ISD_SADDLP_r(VT, RetVT, Op0);
6660 case AArch64ISD::SITOF: return fastEmit_AArch64ISD_SITOF_r(VT, RetVT, Op0);
6661 case AArch64ISD::SQABS: return fastEmit_AArch64ISD_SQABS_r(VT, RetVT, Op0);
6662 case AArch64ISD::SQNEG: return fastEmit_AArch64ISD_SQNEG_r(VT, RetVT, Op0);
6663 case AArch64ISD::SUNPKHI: return fastEmit_AArch64ISD_SUNPKHI_r(VT, RetVT, Op0);
6664 case AArch64ISD::SUNPKLO: return fastEmit_AArch64ISD_SUNPKLO_r(VT, RetVT, Op0);
6665 case AArch64ISD::UADDLP: return fastEmit_AArch64ISD_UADDLP_r(VT, RetVT, Op0);
6666 case AArch64ISD::UITOF: return fastEmit_AArch64ISD_UITOF_r(VT, RetVT, Op0);
6667 case AArch64ISD::UUNPKHI: return fastEmit_AArch64ISD_UUNPKHI_r(VT, RetVT, Op0);
6668 case AArch64ISD::UUNPKLO: return fastEmit_AArch64ISD_UUNPKLO_r(VT, RetVT, Op0);
6669 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
6670 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
6671 case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0);
6672 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0);
6673 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
6674 case ISD::CTLS: return fastEmit_ISD_CTLS_r(VT, RetVT, Op0);
6675 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
6676 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
6677 case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0);
6678 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
6679 case ISD::FCEIL: return fastEmit_ISD_FCEIL_r(VT, RetVT, Op0);
6680 case ISD::FFLOOR: return fastEmit_ISD_FFLOOR_r(VT, RetVT, Op0);
6681 case ISD::FNEARBYINT: return fastEmit_ISD_FNEARBYINT_r(VT, RetVT, Op0);
6682 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
6683 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
6684 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
6685 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
6686 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
6687 case ISD::FRINT: return fastEmit_ISD_FRINT_r(VT, RetVT, Op0);
6688 case ISD::FROUND: return fastEmit_ISD_FROUND_r(VT, RetVT, Op0);
6689 case ISD::FROUNDEVEN: return fastEmit_ISD_FROUNDEVEN_r(VT, RetVT, Op0);
6690 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
6691 case ISD::FTRUNC: return fastEmit_ISD_FTRUNC_r(VT, RetVT, Op0);
6692 case ISD::LLROUND: return fastEmit_ISD_LLROUND_r(VT, RetVT, Op0);
6693 case ISD::LROUND: return fastEmit_ISD_LROUND_r(VT, RetVT, Op0);
6694 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
6695 case ISD::SPLAT_VECTOR: return fastEmit_ISD_SPLAT_VECTOR_r(VT, RetVT, Op0);
6696 case ISD::STRICT_FCEIL: return fastEmit_ISD_STRICT_FCEIL_r(VT, RetVT, Op0);
6697 case ISD::STRICT_FFLOOR: return fastEmit_ISD_STRICT_FFLOOR_r(VT, RetVT, Op0);
6698 case ISD::STRICT_FNEARBYINT: return fastEmit_ISD_STRICT_FNEARBYINT_r(VT, RetVT, Op0);
6699 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0);
6700 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0);
6701 case ISD::STRICT_FP_TO_SINT: return fastEmit_ISD_STRICT_FP_TO_SINT_r(VT, RetVT, Op0);
6702 case ISD::STRICT_FP_TO_UINT: return fastEmit_ISD_STRICT_FP_TO_UINT_r(VT, RetVT, Op0);
6703 case ISD::STRICT_FRINT: return fastEmit_ISD_STRICT_FRINT_r(VT, RetVT, Op0);
6704 case ISD::STRICT_FROUND: return fastEmit_ISD_STRICT_FROUND_r(VT, RetVT, Op0);
6705 case ISD::STRICT_FROUNDEVEN: return fastEmit_ISD_STRICT_FROUNDEVEN_r(VT, RetVT, Op0);
6706 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0);
6707 case ISD::STRICT_FTRUNC: return fastEmit_ISD_STRICT_FTRUNC_r(VT, RetVT, Op0);
6708 case ISD::STRICT_LLROUND: return fastEmit_ISD_STRICT_LLROUND_r(VT, RetVT, Op0);
6709 case ISD::STRICT_LROUND: return fastEmit_ISD_STRICT_LROUND_r(VT, RetVT, Op0);
6710 case ISD::STRICT_SINT_TO_FP: return fastEmit_ISD_STRICT_SINT_TO_FP_r(VT, RetVT, Op0);
6711 case ISD::STRICT_UINT_TO_FP: return fastEmit_ISD_STRICT_UINT_TO_FP_r(VT, RetVT, Op0);
6712 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
6713 case ISD::TRUNCATE_SSAT_S: return fastEmit_ISD_TRUNCATE_SSAT_S_r(VT, RetVT, Op0);
6714 case ISD::TRUNCATE_SSAT_U: return fastEmit_ISD_TRUNCATE_SSAT_U_r(VT, RetVT, Op0);
6715 case ISD::TRUNCATE_USAT_U: return fastEmit_ISD_TRUNCATE_USAT_U_r(VT, RetVT, Op0);
6716 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
6717 case ISD::VECREDUCE_ADD: return fastEmit_ISD_VECREDUCE_ADD_r(VT, RetVT, Op0);
6718 case ISD::VECREDUCE_FADD: return fastEmit_ISD_VECREDUCE_FADD_r(VT, RetVT, Op0);
6719 case ISD::VECREDUCE_FMAX: return fastEmit_ISD_VECREDUCE_FMAX_r(VT, RetVT, Op0);
6720 case ISD::VECREDUCE_FMAXIMUM: return fastEmit_ISD_VECREDUCE_FMAXIMUM_r(VT, RetVT, Op0);
6721 case ISD::VECREDUCE_FMIN: return fastEmit_ISD_VECREDUCE_FMIN_r(VT, RetVT, Op0);
6722 case ISD::VECREDUCE_FMINIMUM: return fastEmit_ISD_VECREDUCE_FMINIMUM_r(VT, RetVT, Op0);
6723 case ISD::VECREDUCE_SMAX: return fastEmit_ISD_VECREDUCE_SMAX_r(VT, RetVT, Op0);
6724 case ISD::VECREDUCE_SMIN: return fastEmit_ISD_VECREDUCE_SMIN_r(VT, RetVT, Op0);
6725 case ISD::VECREDUCE_UMAX: return fastEmit_ISD_VECREDUCE_UMAX_r(VT, RetVT, Op0);
6726 case ISD::VECREDUCE_UMIN: return fastEmit_ISD_VECREDUCE_UMIN_r(VT, RetVT, Op0);
6727 case ISD::VECTOR_REVERSE: return fastEmit_ISD_VECTOR_REVERSE_r(VT, RetVT, Op0);
6728 default: return Register();
6729 }
6730}
6731
6732// FastEmit functions for AArch64ISD::ADDHN.
6733
6734Register fastEmit_AArch64ISD_ADDHN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6735 if (RetVT.SimpleTy != MVT::v8i8)
6736 return Register();
6737 if ((Subtarget->isNeonAvailable())) {
6738 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDHNv8i16_v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
6739 }
6740 return Register();
6741}
6742
6743Register fastEmit_AArch64ISD_ADDHN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6744 if (RetVT.SimpleTy != MVT::v4i16)
6745 return Register();
6746 if ((Subtarget->isNeonAvailable())) {
6747 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDHNv4i32_v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6748 }
6749 return Register();
6750}
6751
6752Register fastEmit_AArch64ISD_ADDHN_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6753 if (RetVT.SimpleTy != MVT::v2i32)
6754 return Register();
6755 if ((Subtarget->isNeonAvailable())) {
6756 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDHNv2i64_v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6757 }
6758 return Register();
6759}
6760
6761Register fastEmit_AArch64ISD_ADDHN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6762 switch (VT.SimpleTy) {
6763 case MVT::v8i16: return fastEmit_AArch64ISD_ADDHN_MVT_v8i16_rr(RetVT, Op0, Op1);
6764 case MVT::v4i32: return fastEmit_AArch64ISD_ADDHN_MVT_v4i32_rr(RetVT, Op0, Op1);
6765 case MVT::v2i64: return fastEmit_AArch64ISD_ADDHN_MVT_v2i64_rr(RetVT, Op0, Op1);
6766 default: return Register();
6767 }
6768}
6769
6770// FastEmit functions for AArch64ISD::ADDP.
6771
6772Register fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6773 if (RetVT.SimpleTy != MVT::v8i8)
6774 return Register();
6775 if ((Subtarget->isNeonAvailable())) {
6776 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
6777 }
6778 return Register();
6779}
6780
6781Register fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6782 if (RetVT.SimpleTy != MVT::v16i8)
6783 return Register();
6784 if ((Subtarget->isNeonAvailable())) {
6785 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
6786 }
6787 return Register();
6788}
6789
6790Register fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6791 if (RetVT.SimpleTy != MVT::v4i16)
6792 return Register();
6793 if ((Subtarget->isNeonAvailable())) {
6794 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6795 }
6796 return Register();
6797}
6798
6799Register fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6800 if (RetVT.SimpleTy != MVT::v8i16)
6801 return Register();
6802 if ((Subtarget->isNeonAvailable())) {
6803 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
6804 }
6805 return Register();
6806}
6807
6808Register fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6809 if (RetVT.SimpleTy != MVT::v2i32)
6810 return Register();
6811 if ((Subtarget->isNeonAvailable())) {
6812 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6813 }
6814 return Register();
6815}
6816
6817Register fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6818 if (RetVT.SimpleTy != MVT::v4i32)
6819 return Register();
6820 if ((Subtarget->isNeonAvailable())) {
6821 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
6822 }
6823 return Register();
6824}
6825
6826Register fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6827 if (RetVT.SimpleTy != MVT::v2i64)
6828 return Register();
6829 if ((Subtarget->isNeonAvailable())) {
6830 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
6831 }
6832 return Register();
6833}
6834
6835Register fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
6836 if (RetVT.SimpleTy != MVT::v4f16)
6837 return Register();
6838 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6839 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6840 }
6841 return Register();
6842}
6843
6844Register fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
6845 if (RetVT.SimpleTy != MVT::v8f16)
6846 return Register();
6847 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6848 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
6849 }
6850 return Register();
6851}
6852
6853Register fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
6854 if (RetVT.SimpleTy != MVT::v2f32)
6855 return Register();
6856 if ((Subtarget->isNeonAvailable())) {
6857 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6858 }
6859 return Register();
6860}
6861
6862Register fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
6863 if (RetVT.SimpleTy != MVT::v4f32)
6864 return Register();
6865 if ((Subtarget->isNeonAvailable())) {
6866 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
6867 }
6868 return Register();
6869}
6870
6871Register fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
6872 if (RetVT.SimpleTy != MVT::v2f64)
6873 return Register();
6874 if ((Subtarget->isNeonAvailable())) {
6875 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
6876 }
6877 return Register();
6878}
6879
6880Register fastEmit_AArch64ISD_ADDP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6881 switch (VT.SimpleTy) {
6882 case MVT::v8i8: return fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(RetVT, Op0, Op1);
6883 case MVT::v16i8: return fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(RetVT, Op0, Op1);
6884 case MVT::v4i16: return fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(RetVT, Op0, Op1);
6885 case MVT::v8i16: return fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(RetVT, Op0, Op1);
6886 case MVT::v2i32: return fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(RetVT, Op0, Op1);
6887 case MVT::v4i32: return fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(RetVT, Op0, Op1);
6888 case MVT::v2i64: return fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(RetVT, Op0, Op1);
6889 case MVT::v4f16: return fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(RetVT, Op0, Op1);
6890 case MVT::v8f16: return fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(RetVT, Op0, Op1);
6891 case MVT::v2f32: return fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(RetVT, Op0, Op1);
6892 case MVT::v4f32: return fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(RetVT, Op0, Op1);
6893 case MVT::v2f64: return fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(RetVT, Op0, Op1);
6894 default: return Register();
6895 }
6896}
6897
6898// FastEmit functions for AArch64ISD::BIC.
6899
6900Register fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6901 if (RetVT.SimpleTy != MVT::nxv16i8)
6902 return Register();
6903 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6904 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6905 }
6906 return Register();
6907}
6908
6909Register fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6910 if (RetVT.SimpleTy != MVT::nxv8i16)
6911 return Register();
6912 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6913 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6914 }
6915 return Register();
6916}
6917
6918Register fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6919 if (RetVT.SimpleTy != MVT::nxv4i32)
6920 return Register();
6921 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6922 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6923 }
6924 return Register();
6925}
6926
6927Register fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6928 if (RetVT.SimpleTy != MVT::nxv2i64)
6929 return Register();
6930 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6931 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6932 }
6933 return Register();
6934}
6935
6936Register fastEmit_AArch64ISD_BIC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6937 switch (VT.SimpleTy) {
6938 case MVT::nxv16i8: return fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(RetVT, Op0, Op1);
6939 case MVT::nxv8i16: return fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(RetVT, Op0, Op1);
6940 case MVT::nxv4i32: return fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(RetVT, Op0, Op1);
6941 case MVT::nxv2i64: return fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(RetVT, Op0, Op1);
6942 default: return Register();
6943 }
6944}
6945
6946// FastEmit functions for AArch64ISD::CMTST.
6947
6948Register fastEmit_AArch64ISD_CMTST_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6949 if (RetVT.SimpleTy != MVT::v8i8)
6950 return Register();
6951 if ((Subtarget->isNeonAvailable())) {
6952 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
6953 }
6954 return Register();
6955}
6956
6957Register fastEmit_AArch64ISD_CMTST_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6958 if (RetVT.SimpleTy != MVT::v16i8)
6959 return Register();
6960 if ((Subtarget->isNeonAvailable())) {
6961 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
6962 }
6963 return Register();
6964}
6965
6966Register fastEmit_AArch64ISD_CMTST_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6967 if (RetVT.SimpleTy != MVT::v4i16)
6968 return Register();
6969 if ((Subtarget->isNeonAvailable())) {
6970 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6971 }
6972 return Register();
6973}
6974
6975Register fastEmit_AArch64ISD_CMTST_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6976 if (RetVT.SimpleTy != MVT::v8i16)
6977 return Register();
6978 if ((Subtarget->isNeonAvailable())) {
6979 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
6980 }
6981 return Register();
6982}
6983
6984Register fastEmit_AArch64ISD_CMTST_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6985 if (RetVT.SimpleTy != MVT::v2i32)
6986 return Register();
6987 if ((Subtarget->isNeonAvailable())) {
6988 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6989 }
6990 return Register();
6991}
6992
6993Register fastEmit_AArch64ISD_CMTST_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6994 if (RetVT.SimpleTy != MVT::v4i32)
6995 return Register();
6996 if ((Subtarget->isNeonAvailable())) {
6997 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
6998 }
6999 return Register();
7000}
7001
7002Register fastEmit_AArch64ISD_CMTST_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7003 if (RetVT.SimpleTy != MVT::v1i64)
7004 return Register();
7005 if ((Subtarget->isNeonAvailable())) {
7006 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7007 }
7008 return Register();
7009}
7010
7011Register fastEmit_AArch64ISD_CMTST_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7012 if (RetVT.SimpleTy != MVT::v2i64)
7013 return Register();
7014 if ((Subtarget->isNeonAvailable())) {
7015 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7016 }
7017 return Register();
7018}
7019
7020Register fastEmit_AArch64ISD_CMTST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7021 switch (VT.SimpleTy) {
7022 case MVT::v8i8: return fastEmit_AArch64ISD_CMTST_MVT_v8i8_rr(RetVT, Op0, Op1);
7023 case MVT::v16i8: return fastEmit_AArch64ISD_CMTST_MVT_v16i8_rr(RetVT, Op0, Op1);
7024 case MVT::v4i16: return fastEmit_AArch64ISD_CMTST_MVT_v4i16_rr(RetVT, Op0, Op1);
7025 case MVT::v8i16: return fastEmit_AArch64ISD_CMTST_MVT_v8i16_rr(RetVT, Op0, Op1);
7026 case MVT::v2i32: return fastEmit_AArch64ISD_CMTST_MVT_v2i32_rr(RetVT, Op0, Op1);
7027 case MVT::v4i32: return fastEmit_AArch64ISD_CMTST_MVT_v4i32_rr(RetVT, Op0, Op1);
7028 case MVT::v1i64: return fastEmit_AArch64ISD_CMTST_MVT_v1i64_rr(RetVT, Op0, Op1);
7029 case MVT::v2i64: return fastEmit_AArch64ISD_CMTST_MVT_v2i64_rr(RetVT, Op0, Op1);
7030 default: return Register();
7031 }
7032}
7033
7034// FastEmit functions for AArch64ISD::FCMEQ.
7035
7036Register fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7037 if (RetVT.SimpleTy != MVT::i32)
7038 return Register();
7039 if ((Subtarget->isNeonAvailable())) {
7040 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7041 }
7042 return Register();
7043}
7044
7045Register fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7046 if (RetVT.SimpleTy != MVT::i64)
7047 return Register();
7048 if ((Subtarget->isNeonAvailable())) {
7049 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7050 }
7051 return Register();
7052}
7053
7054Register fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7055 if (RetVT.SimpleTy != MVT::v4i16)
7056 return Register();
7057 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7058 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7059 }
7060 return Register();
7061}
7062
7063Register fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7064 if (RetVT.SimpleTy != MVT::v8i16)
7065 return Register();
7066 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7067 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7068 }
7069 return Register();
7070}
7071
7072Register fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7073 if (RetVT.SimpleTy != MVT::v2i32)
7074 return Register();
7075 if ((Subtarget->isNeonAvailable())) {
7076 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7077 }
7078 return Register();
7079}
7080
7081Register fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7082 if (RetVT.SimpleTy != MVT::v4i32)
7083 return Register();
7084 if ((Subtarget->isNeonAvailable())) {
7085 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7086 }
7087 return Register();
7088}
7089
7090Register fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7091 if (RetVT.SimpleTy != MVT::v1i64)
7092 return Register();
7093 if ((Subtarget->isNeonAvailable())) {
7094 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7095 }
7096 return Register();
7097}
7098
7099Register fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7100 if (RetVT.SimpleTy != MVT::v2i64)
7101 return Register();
7102 if ((Subtarget->isNeonAvailable())) {
7103 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7104 }
7105 return Register();
7106}
7107
7108Register fastEmit_AArch64ISD_FCMEQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7109 switch (VT.SimpleTy) {
7110 case MVT::f32: return fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(RetVT, Op0, Op1);
7111 case MVT::f64: return fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(RetVT, Op0, Op1);
7112 case MVT::v4f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(RetVT, Op0, Op1);
7113 case MVT::v8f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(RetVT, Op0, Op1);
7114 case MVT::v2f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(RetVT, Op0, Op1);
7115 case MVT::v4f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(RetVT, Op0, Op1);
7116 case MVT::v1f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(RetVT, Op0, Op1);
7117 case MVT::v2f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(RetVT, Op0, Op1);
7118 default: return Register();
7119 }
7120}
7121
7122// FastEmit functions for AArch64ISD::FCMGE.
7123
7124Register fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7125 if (RetVT.SimpleTy != MVT::i32)
7126 return Register();
7127 if ((Subtarget->isNeonAvailable())) {
7128 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7129 }
7130 return Register();
7131}
7132
7133Register fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7134 if (RetVT.SimpleTy != MVT::i64)
7135 return Register();
7136 if ((Subtarget->isNeonAvailable())) {
7137 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7138 }
7139 return Register();
7140}
7141
7142Register fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7143 if (RetVT.SimpleTy != MVT::v4i16)
7144 return Register();
7145 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7146 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7147 }
7148 return Register();
7149}
7150
7151Register fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7152 if (RetVT.SimpleTy != MVT::v8i16)
7153 return Register();
7154 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7155 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7156 }
7157 return Register();
7158}
7159
7160Register fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7161 if (RetVT.SimpleTy != MVT::v2i32)
7162 return Register();
7163 if ((Subtarget->isNeonAvailable())) {
7164 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7165 }
7166 return Register();
7167}
7168
7169Register fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7170 if (RetVT.SimpleTy != MVT::v4i32)
7171 return Register();
7172 if ((Subtarget->isNeonAvailable())) {
7173 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7174 }
7175 return Register();
7176}
7177
7178Register fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7179 if (RetVT.SimpleTy != MVT::v1i64)
7180 return Register();
7181 if ((Subtarget->isNeonAvailable())) {
7182 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7183 }
7184 return Register();
7185}
7186
7187Register fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7188 if (RetVT.SimpleTy != MVT::v2i64)
7189 return Register();
7190 if ((Subtarget->isNeonAvailable())) {
7191 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7192 }
7193 return Register();
7194}
7195
7196Register fastEmit_AArch64ISD_FCMGE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7197 switch (VT.SimpleTy) {
7198 case MVT::f32: return fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(RetVT, Op0, Op1);
7199 case MVT::f64: return fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(RetVT, Op0, Op1);
7200 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(RetVT, Op0, Op1);
7201 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(RetVT, Op0, Op1);
7202 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(RetVT, Op0, Op1);
7203 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(RetVT, Op0, Op1);
7204 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(RetVT, Op0, Op1);
7205 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(RetVT, Op0, Op1);
7206 default: return Register();
7207 }
7208}
7209
7210// FastEmit functions for AArch64ISD::FCMGT.
7211
7212Register fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7213 if (RetVT.SimpleTy != MVT::i32)
7214 return Register();
7215 if ((Subtarget->isNeonAvailable())) {
7216 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7217 }
7218 return Register();
7219}
7220
7221Register fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7222 if (RetVT.SimpleTy != MVT::i64)
7223 return Register();
7224 if ((Subtarget->isNeonAvailable())) {
7225 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7226 }
7227 return Register();
7228}
7229
7230Register fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7231 if (RetVT.SimpleTy != MVT::v4i16)
7232 return Register();
7233 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7234 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7235 }
7236 return Register();
7237}
7238
7239Register fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7240 if (RetVT.SimpleTy != MVT::v8i16)
7241 return Register();
7242 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7243 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7244 }
7245 return Register();
7246}
7247
7248Register fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7249 if (RetVT.SimpleTy != MVT::v2i32)
7250 return Register();
7251 if ((Subtarget->isNeonAvailable())) {
7252 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7253 }
7254 return Register();
7255}
7256
7257Register fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7258 if (RetVT.SimpleTy != MVT::v4i32)
7259 return Register();
7260 if ((Subtarget->isNeonAvailable())) {
7261 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7262 }
7263 return Register();
7264}
7265
7266Register fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7267 if (RetVT.SimpleTy != MVT::v1i64)
7268 return Register();
7269 if ((Subtarget->isNeonAvailable())) {
7270 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7271 }
7272 return Register();
7273}
7274
7275Register fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7276 if (RetVT.SimpleTy != MVT::v2i64)
7277 return Register();
7278 if ((Subtarget->isNeonAvailable())) {
7279 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7280 }
7281 return Register();
7282}
7283
7284Register fastEmit_AArch64ISD_FCMGT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7285 switch (VT.SimpleTy) {
7286 case MVT::f32: return fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(RetVT, Op0, Op1);
7287 case MVT::f64: return fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(RetVT, Op0, Op1);
7288 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(RetVT, Op0, Op1);
7289 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(RetVT, Op0, Op1);
7290 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(RetVT, Op0, Op1);
7291 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(RetVT, Op0, Op1);
7292 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(RetVT, Op0, Op1);
7293 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(RetVT, Op0, Op1);
7294 default: return Register();
7295 }
7296}
7297
7298// FastEmit functions for AArch64ISD::FCMP.
7299
7300Register fastEmit_AArch64ISD_FCMP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
7301 if (RetVT.SimpleTy != MVT::i32)
7302 return Register();
7303 if ((Subtarget->hasFullFP16())) {
7304 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
7305 }
7306 return Register();
7307}
7308
7309Register fastEmit_AArch64ISD_FCMP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7310 if (RetVT.SimpleTy != MVT::i32)
7311 return Register();
7312 if ((Subtarget->hasFPARMv8())) {
7313 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
7314 }
7315 return Register();
7316}
7317
7318Register fastEmit_AArch64ISD_FCMP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7319 if (RetVT.SimpleTy != MVT::i32)
7320 return Register();
7321 if ((Subtarget->hasFPARMv8())) {
7322 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
7323 }
7324 return Register();
7325}
7326
7327Register fastEmit_AArch64ISD_FCMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7328 switch (VT.SimpleTy) {
7329 case MVT::f16: return fastEmit_AArch64ISD_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
7330 case MVT::f32: return fastEmit_AArch64ISD_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
7331 case MVT::f64: return fastEmit_AArch64ISD_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
7332 default: return Register();
7333 }
7334}
7335
7336// FastEmit functions for AArch64ISD::FRECPS.
7337
7338Register fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7339 if (RetVT.SimpleTy != MVT::f32)
7340 return Register();
7341 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7342}
7343
7344Register fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7345 if (RetVT.SimpleTy != MVT::f64)
7346 return Register();
7347 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7348}
7349
7350Register fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7351 if (RetVT.SimpleTy != MVT::v2f32)
7352 return Register();
7353 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7354}
7355
7356Register fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7357 if (RetVT.SimpleTy != MVT::v4f32)
7358 return Register();
7359 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7360}
7361
7362Register fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7363 if (RetVT.SimpleTy != MVT::v2f64)
7364 return Register();
7365 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7366}
7367
7368Register fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7369 if (RetVT.SimpleTy != MVT::nxv8f16)
7370 return Register();
7371 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7372 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7373 }
7374 return Register();
7375}
7376
7377Register fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7378 if (RetVT.SimpleTy != MVT::nxv4f32)
7379 return Register();
7380 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7381 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7382 }
7383 return Register();
7384}
7385
7386Register fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7387 if (RetVT.SimpleTy != MVT::nxv2f64)
7388 return Register();
7389 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7390 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7391 }
7392 return Register();
7393}
7394
7395Register fastEmit_AArch64ISD_FRECPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7396 switch (VT.SimpleTy) {
7397 case MVT::f32: return fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(RetVT, Op0, Op1);
7398 case MVT::f64: return fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(RetVT, Op0, Op1);
7399 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(RetVT, Op0, Op1);
7400 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(RetVT, Op0, Op1);
7401 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(RetVT, Op0, Op1);
7402 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(RetVT, Op0, Op1);
7403 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(RetVT, Op0, Op1);
7404 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(RetVT, Op0, Op1);
7405 default: return Register();
7406 }
7407}
7408
7409// FastEmit functions for AArch64ISD::FRSQRTS.
7410
7411Register fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7412 if (RetVT.SimpleTy != MVT::f32)
7413 return Register();
7414 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7415}
7416
7417Register fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7418 if (RetVT.SimpleTy != MVT::f64)
7419 return Register();
7420 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7421}
7422
7423Register fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7424 if (RetVT.SimpleTy != MVT::v2f32)
7425 return Register();
7426 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7427}
7428
7429Register fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7430 if (RetVT.SimpleTy != MVT::v4f32)
7431 return Register();
7432 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7433}
7434
7435Register fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7436 if (RetVT.SimpleTy != MVT::v2f64)
7437 return Register();
7438 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7439}
7440
7441Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7442 if (RetVT.SimpleTy != MVT::nxv8f16)
7443 return Register();
7444 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7445 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7446 }
7447 return Register();
7448}
7449
7450Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7451 if (RetVT.SimpleTy != MVT::nxv4f32)
7452 return Register();
7453 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7454 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7455 }
7456 return Register();
7457}
7458
7459Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7460 if (RetVT.SimpleTy != MVT::nxv2f64)
7461 return Register();
7462 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7463 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7464 }
7465 return Register();
7466}
7467
7468Register fastEmit_AArch64ISD_FRSQRTS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7469 switch (VT.SimpleTy) {
7470 case MVT::f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(RetVT, Op0, Op1);
7471 case MVT::f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(RetVT, Op0, Op1);
7472 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(RetVT, Op0, Op1);
7473 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(RetVT, Op0, Op1);
7474 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(RetVT, Op0, Op1);
7475 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(RetVT, Op0, Op1);
7476 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(RetVT, Op0, Op1);
7477 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(RetVT, Op0, Op1);
7478 default: return Register();
7479 }
7480}
7481
7482// FastEmit functions for AArch64ISD::PMULL.
7483
7484Register fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7485 if (RetVT.SimpleTy != MVT::v8i16)
7486 return Register();
7487 if ((Subtarget->isNeonAvailable())) {
7488 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULLv8i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
7489 }
7490 return Register();
7491}
7492
7493Register fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7494 if (RetVT.SimpleTy != MVT::v16i8)
7495 return Register();
7496 if ((Subtarget->hasAES())) {
7497 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULLv1i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7498 }
7499 return Register();
7500}
7501
7502Register fastEmit_AArch64ISD_PMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7503 switch (VT.SimpleTy) {
7504 case MVT::v8i8: return fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
7505 case MVT::v1i64: return fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(RetVT, Op0, Op1);
7506 default: return Register();
7507 }
7508}
7509
7510// FastEmit functions for AArch64ISD::PTEST.
7511
7512Register fastEmit_AArch64ISD_PTEST_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7513 if (RetVT.SimpleTy != MVT::i32)
7514 return Register();
7515 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7516 return fastEmitInst_rr(MachineInstOpcode: AArch64::PTEST_PP, RC: &AArch64::PPRRegClass, Op0, Op1);
7517 }
7518 return Register();
7519}
7520
7521Register fastEmit_AArch64ISD_PTEST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7522 switch (VT.SimpleTy) {
7523 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7524 default: return Register();
7525 }
7526}
7527
7528// FastEmit functions for AArch64ISD::PTEST_ANY.
7529
7530Register fastEmit_AArch64ISD_PTEST_ANY_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7531 if (RetVT.SimpleTy != MVT::i32)
7532 return Register();
7533 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7534 return fastEmitInst_rr(MachineInstOpcode: AArch64::PTEST_PP_ANY, RC: &AArch64::PPRRegClass, Op0, Op1);
7535 }
7536 return Register();
7537}
7538
7539Register fastEmit_AArch64ISD_PTEST_ANY_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7540 switch (VT.SimpleTy) {
7541 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_ANY_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7542 default: return Register();
7543 }
7544}
7545
7546// FastEmit functions for AArch64ISD::PTEST_FIRST.
7547
7548Register fastEmit_AArch64ISD_PTEST_FIRST_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7549 if (RetVT.SimpleTy != MVT::i32)
7550 return Register();
7551 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7552 return fastEmitInst_rr(MachineInstOpcode: AArch64::PTEST_PP_FIRST, RC: &AArch64::PPRRegClass, Op0, Op1);
7553 }
7554 return Register();
7555}
7556
7557Register fastEmit_AArch64ISD_PTEST_FIRST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7558 switch (VT.SimpleTy) {
7559 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_FIRST_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7560 default: return Register();
7561 }
7562}
7563
7564// FastEmit functions for AArch64ISD::SMULL.
7565
7566Register fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7567 if (RetVT.SimpleTy != MVT::v8i16)
7568 return Register();
7569 if ((Subtarget->isNeonAvailable())) {
7570 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv8i8_v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7571 }
7572 return Register();
7573}
7574
7575Register fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7576 if (RetVT.SimpleTy != MVT::v4i32)
7577 return Register();
7578 if ((Subtarget->isNeonAvailable())) {
7579 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv4i16_v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7580 }
7581 return Register();
7582}
7583
7584Register fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7585 if (RetVT.SimpleTy != MVT::v2i64)
7586 return Register();
7587 if ((Subtarget->isNeonAvailable())) {
7588 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv2i32_v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7589 }
7590 return Register();
7591}
7592
7593Register fastEmit_AArch64ISD_SMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7594 switch (VT.SimpleTy) {
7595 case MVT::v8i8: return fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
7596 case MVT::v4i16: return fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(RetVT, Op0, Op1);
7597 case MVT::v2i32: return fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(RetVT, Op0, Op1);
7598 default: return Register();
7599 }
7600}
7601
7602// FastEmit functions for AArch64ISD::SQADD.
7603
7604Register fastEmit_AArch64ISD_SQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7605 if (RetVT.SimpleTy != MVT::f32)
7606 return Register();
7607 if ((Subtarget->isNeonAvailable())) {
7608 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7609 }
7610 return Register();
7611}
7612
7613Register fastEmit_AArch64ISD_SQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7614 if (RetVT.SimpleTy != MVT::f64)
7615 return Register();
7616 if ((Subtarget->isNeonAvailable())) {
7617 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7618 }
7619 return Register();
7620}
7621
7622Register fastEmit_AArch64ISD_SQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7623 switch (VT.SimpleTy) {
7624 case MVT::f32: return fastEmit_AArch64ISD_SQADD_MVT_f32_rr(RetVT, Op0, Op1);
7625 case MVT::f64: return fastEmit_AArch64ISD_SQADD_MVT_f64_rr(RetVT, Op0, Op1);
7626 default: return Register();
7627 }
7628}
7629
7630// FastEmit functions for AArch64ISD::SQDMULH.
7631
7632Register fastEmit_AArch64ISD_SQDMULH_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7633 if (RetVT.SimpleTy != MVT::f32)
7634 return Register();
7635 if ((Subtarget->isNeonAvailable())) {
7636 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7637 }
7638 return Register();
7639}
7640
7641Register fastEmit_AArch64ISD_SQDMULH_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7642 if (RetVT.SimpleTy != MVT::v4i16)
7643 return Register();
7644 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7645}
7646
7647Register fastEmit_AArch64ISD_SQDMULH_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7648 if (RetVT.SimpleTy != MVT::v8i16)
7649 return Register();
7650 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7651}
7652
7653Register fastEmit_AArch64ISD_SQDMULH_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7654 if (RetVT.SimpleTy != MVT::v2i32)
7655 return Register();
7656 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7657}
7658
7659Register fastEmit_AArch64ISD_SQDMULH_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7660 if (RetVT.SimpleTy != MVT::v4i32)
7661 return Register();
7662 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7663}
7664
7665Register fastEmit_AArch64ISD_SQDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7666 switch (VT.SimpleTy) {
7667 case MVT::f32: return fastEmit_AArch64ISD_SQDMULH_MVT_f32_rr(RetVT, Op0, Op1);
7668 case MVT::v4i16: return fastEmit_AArch64ISD_SQDMULH_MVT_v4i16_rr(RetVT, Op0, Op1);
7669 case MVT::v8i16: return fastEmit_AArch64ISD_SQDMULH_MVT_v8i16_rr(RetVT, Op0, Op1);
7670 case MVT::v2i32: return fastEmit_AArch64ISD_SQDMULH_MVT_v2i32_rr(RetVT, Op0, Op1);
7671 case MVT::v4i32: return fastEmit_AArch64ISD_SQDMULH_MVT_v4i32_rr(RetVT, Op0, Op1);
7672 default: return Register();
7673 }
7674}
7675
7676// FastEmit functions for AArch64ISD::SQDMULL.
7677
7678Register fastEmit_AArch64ISD_SQDMULL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7679 if (RetVT.SimpleTy != MVT::f64)
7680 return Register();
7681 if ((Subtarget->isNeonAvailable())) {
7682 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULLi32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7683 }
7684 return Register();
7685}
7686
7687Register fastEmit_AArch64ISD_SQDMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7688 switch (VT.SimpleTy) {
7689 case MVT::f32: return fastEmit_AArch64ISD_SQDMULL_MVT_f32_rr(RetVT, Op0, Op1);
7690 default: return Register();
7691 }
7692}
7693
7694// FastEmit functions for AArch64ISD::SQRDMULH.
7695
7696Register fastEmit_AArch64ISD_SQRDMULH_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7697 if (RetVT.SimpleTy != MVT::f32)
7698 return Register();
7699 if ((Subtarget->isNeonAvailable())) {
7700 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRDMULHv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7701 }
7702 return Register();
7703}
7704
7705Register fastEmit_AArch64ISD_SQRDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7706 switch (VT.SimpleTy) {
7707 case MVT::f32: return fastEmit_AArch64ISD_SQRDMULH_MVT_f32_rr(RetVT, Op0, Op1);
7708 default: return Register();
7709 }
7710}
7711
7712// FastEmit functions for AArch64ISD::SQRSHL.
7713
7714Register fastEmit_AArch64ISD_SQRSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7715 if (RetVT.SimpleTy != MVT::f32)
7716 return Register();
7717 if ((Subtarget->isNeonAvailable())) {
7718 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7719 }
7720 return Register();
7721}
7722
7723Register fastEmit_AArch64ISD_SQRSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7724 if (RetVT.SimpleTy != MVT::f64)
7725 return Register();
7726 if ((Subtarget->isNeonAvailable())) {
7727 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7728 }
7729 return Register();
7730}
7731
7732Register fastEmit_AArch64ISD_SQRSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7733 switch (VT.SimpleTy) {
7734 case MVT::f32: return fastEmit_AArch64ISD_SQRSHL_MVT_f32_rr(RetVT, Op0, Op1);
7735 case MVT::f64: return fastEmit_AArch64ISD_SQRSHL_MVT_f64_rr(RetVT, Op0, Op1);
7736 default: return Register();
7737 }
7738}
7739
7740// FastEmit functions for AArch64ISD::SQSHL.
7741
7742Register fastEmit_AArch64ISD_SQSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7743 if (RetVT.SimpleTy != MVT::f32)
7744 return Register();
7745 if ((Subtarget->isNeonAvailable())) {
7746 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7747 }
7748 return Register();
7749}
7750
7751Register fastEmit_AArch64ISD_SQSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7752 if (RetVT.SimpleTy != MVT::f64)
7753 return Register();
7754 if ((Subtarget->isNeonAvailable())) {
7755 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7756 }
7757 return Register();
7758}
7759
7760Register fastEmit_AArch64ISD_SQSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7761 switch (VT.SimpleTy) {
7762 case MVT::f32: return fastEmit_AArch64ISD_SQSHL_MVT_f32_rr(RetVT, Op0, Op1);
7763 case MVT::f64: return fastEmit_AArch64ISD_SQSHL_MVT_f64_rr(RetVT, Op0, Op1);
7764 default: return Register();
7765 }
7766}
7767
7768// FastEmit functions for AArch64ISD::SQSUB.
7769
7770Register fastEmit_AArch64ISD_SQSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7771 if (RetVT.SimpleTy != MVT::f32)
7772 return Register();
7773 if ((Subtarget->isNeonAvailable())) {
7774 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7775 }
7776 return Register();
7777}
7778
7779Register fastEmit_AArch64ISD_SQSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7780 if (RetVT.SimpleTy != MVT::f64)
7781 return Register();
7782 if ((Subtarget->isNeonAvailable())) {
7783 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7784 }
7785 return Register();
7786}
7787
7788Register fastEmit_AArch64ISD_SQSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7789 switch (VT.SimpleTy) {
7790 case MVT::f32: return fastEmit_AArch64ISD_SQSUB_MVT_f32_rr(RetVT, Op0, Op1);
7791 case MVT::f64: return fastEmit_AArch64ISD_SQSUB_MVT_f64_rr(RetVT, Op0, Op1);
7792 default: return Register();
7793 }
7794}
7795
7796// FastEmit functions for AArch64ISD::STRICT_FCMP.
7797
7798Register fastEmit_AArch64ISD_STRICT_FCMP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
7799 if (RetVT.SimpleTy != MVT::i32)
7800 return Register();
7801 if ((Subtarget->hasFullFP16())) {
7802 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
7803 }
7804 return Register();
7805}
7806
7807Register fastEmit_AArch64ISD_STRICT_FCMP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7808 if (RetVT.SimpleTy != MVT::i32)
7809 return Register();
7810 if ((Subtarget->hasFPARMv8())) {
7811 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
7812 }
7813 return Register();
7814}
7815
7816Register fastEmit_AArch64ISD_STRICT_FCMP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7817 if (RetVT.SimpleTy != MVT::i32)
7818 return Register();
7819 if ((Subtarget->hasFPARMv8())) {
7820 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
7821 }
7822 return Register();
7823}
7824
7825Register fastEmit_AArch64ISD_STRICT_FCMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7826 switch (VT.SimpleTy) {
7827 case MVT::f16: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
7828 case MVT::f32: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
7829 case MVT::f64: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
7830 default: return Register();
7831 }
7832}
7833
7834// FastEmit functions for AArch64ISD::STRICT_FCMPE.
7835
7836Register fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
7837 if (RetVT.SimpleTy != MVT::i32)
7838 return Register();
7839 if ((Subtarget->hasFullFP16())) {
7840 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPEHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
7841 }
7842 return Register();
7843}
7844
7845Register fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7846 if (RetVT.SimpleTy != MVT::i32)
7847 return Register();
7848 if ((Subtarget->hasFPARMv8())) {
7849 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPESrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
7850 }
7851 return Register();
7852}
7853
7854Register fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7855 if (RetVT.SimpleTy != MVT::i32)
7856 return Register();
7857 if ((Subtarget->hasFPARMv8())) {
7858 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPEDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
7859 }
7860 return Register();
7861}
7862
7863Register fastEmit_AArch64ISD_STRICT_FCMPE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7864 switch (VT.SimpleTy) {
7865 case MVT::f16: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f16_rr(RetVT, Op0, Op1);
7866 case MVT::f32: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f32_rr(RetVT, Op0, Op1);
7867 case MVT::f64: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f64_rr(RetVT, Op0, Op1);
7868 default: return Register();
7869 }
7870}
7871
7872// FastEmit functions for AArch64ISD::SUQADD.
7873
7874Register fastEmit_AArch64ISD_SUQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7875 if (RetVT.SimpleTy != MVT::f32)
7876 return Register();
7877 if ((Subtarget->isNeonAvailable())) {
7878 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7879 }
7880 return Register();
7881}
7882
7883Register fastEmit_AArch64ISD_SUQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7884 if (RetVT.SimpleTy != MVT::f64)
7885 return Register();
7886 if ((Subtarget->isNeonAvailable())) {
7887 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7888 }
7889 return Register();
7890}
7891
7892Register fastEmit_AArch64ISD_SUQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7893 switch (VT.SimpleTy) {
7894 case MVT::f32: return fastEmit_AArch64ISD_SUQADD_MVT_f32_rr(RetVT, Op0, Op1);
7895 case MVT::f64: return fastEmit_AArch64ISD_SUQADD_MVT_f64_rr(RetVT, Op0, Op1);
7896 default: return Register();
7897 }
7898}
7899
7900// FastEmit functions for AArch64ISD::TBL.
7901
7902Register fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7903 if (RetVT.SimpleTy != MVT::nxv16i8)
7904 return Register();
7905 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7906 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
7907 }
7908 return Register();
7909}
7910
7911Register fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7912 if (RetVT.SimpleTy != MVT::nxv8i16)
7913 return Register();
7914 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7915 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7916 }
7917 return Register();
7918}
7919
7920Register fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7921 if (RetVT.SimpleTy != MVT::nxv4i32)
7922 return Register();
7923 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7924 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7925 }
7926 return Register();
7927}
7928
7929Register fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7930 if (RetVT.SimpleTy != MVT::nxv2i64)
7931 return Register();
7932 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7933 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7934 }
7935 return Register();
7936}
7937
7938Register fastEmit_AArch64ISD_TBL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7939 switch (VT.SimpleTy) {
7940 case MVT::nxv16i8: return fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(RetVT, Op0, Op1);
7941 case MVT::nxv8i16: return fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(RetVT, Op0, Op1);
7942 case MVT::nxv4i32: return fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(RetVT, Op0, Op1);
7943 case MVT::nxv2i64: return fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(RetVT, Op0, Op1);
7944 default: return Register();
7945 }
7946}
7947
7948// FastEmit functions for AArch64ISD::TRN1.
7949
7950Register fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7951 if (RetVT.SimpleTy != MVT::v8i8)
7952 return Register();
7953 if ((Subtarget->isNeonAvailable())) {
7954 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
7955 }
7956 return Register();
7957}
7958
7959Register fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7960 if (RetVT.SimpleTy != MVT::v16i8)
7961 return Register();
7962 if ((Subtarget->isNeonAvailable())) {
7963 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
7964 }
7965 return Register();
7966}
7967
7968Register fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7969 if (RetVT.SimpleTy != MVT::v4i16)
7970 return Register();
7971 if ((Subtarget->isNeonAvailable())) {
7972 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7973 }
7974 return Register();
7975}
7976
7977Register fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7978 if (RetVT.SimpleTy != MVT::v8i16)
7979 return Register();
7980 if ((Subtarget->isNeonAvailable())) {
7981 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7982 }
7983 return Register();
7984}
7985
7986Register fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7987 if (RetVT.SimpleTy != MVT::v2i32)
7988 return Register();
7989 if ((Subtarget->isNeonAvailable())) {
7990 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7991 }
7992 return Register();
7993}
7994
7995Register fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7996 if (RetVT.SimpleTy != MVT::v4i32)
7997 return Register();
7998 if ((Subtarget->isNeonAvailable())) {
7999 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8000 }
8001 return Register();
8002}
8003
8004Register fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8005 if (RetVT.SimpleTy != MVT::v2i64)
8006 return Register();
8007 if ((Subtarget->isNeonAvailable())) {
8008 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8009 }
8010 return Register();
8011}
8012
8013Register fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8014 if (RetVT.SimpleTy != MVT::v4f16)
8015 return Register();
8016 if ((Subtarget->isNeonAvailable())) {
8017 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8018 }
8019 return Register();
8020}
8021
8022Register fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8023 if (RetVT.SimpleTy != MVT::v8f16)
8024 return Register();
8025 if ((Subtarget->isNeonAvailable())) {
8026 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8027 }
8028 return Register();
8029}
8030
8031Register fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8032 if (RetVT.SimpleTy != MVT::v4bf16)
8033 return Register();
8034 if ((Subtarget->isNeonAvailable())) {
8035 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8036 }
8037 return Register();
8038}
8039
8040Register fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8041 if (RetVT.SimpleTy != MVT::v8bf16)
8042 return Register();
8043 if ((Subtarget->isNeonAvailable())) {
8044 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8045 }
8046 return Register();
8047}
8048
8049Register fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8050 if (RetVT.SimpleTy != MVT::v2f32)
8051 return Register();
8052 if ((Subtarget->isNeonAvailable())) {
8053 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8054 }
8055 return Register();
8056}
8057
8058Register fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8059 if (RetVT.SimpleTy != MVT::v4f32)
8060 return Register();
8061 if ((Subtarget->isNeonAvailable())) {
8062 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8063 }
8064 return Register();
8065}
8066
8067Register fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8068 if (RetVT.SimpleTy != MVT::v2f64)
8069 return Register();
8070 if ((Subtarget->isNeonAvailable())) {
8071 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8072 }
8073 return Register();
8074}
8075
8076Register fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8077 if (RetVT.SimpleTy != MVT::nxv2i1)
8078 return Register();
8079 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8080 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8081 }
8082 return Register();
8083}
8084
8085Register fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8086 if (RetVT.SimpleTy != MVT::nxv4i1)
8087 return Register();
8088 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8089 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8090 }
8091 return Register();
8092}
8093
8094Register fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8095 if (RetVT.SimpleTy != MVT::nxv8i1)
8096 return Register();
8097 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8098 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8099 }
8100 return Register();
8101}
8102
8103Register fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8104 if (RetVT.SimpleTy != MVT::nxv16i1)
8105 return Register();
8106 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8107 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8108 }
8109 return Register();
8110}
8111
8112Register fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8113 if (RetVT.SimpleTy != MVT::nxv16i8)
8114 return Register();
8115 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8116 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8117 }
8118 return Register();
8119}
8120
8121Register fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8122 if (RetVT.SimpleTy != MVT::nxv8i16)
8123 return Register();
8124 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8125 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8126 }
8127 return Register();
8128}
8129
8130Register fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8131 if (RetVT.SimpleTy != MVT::nxv4i32)
8132 return Register();
8133 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8134 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8135 }
8136 return Register();
8137}
8138
8139Register fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8140 if (RetVT.SimpleTy != MVT::nxv2i64)
8141 return Register();
8142 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8143 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8144 }
8145 return Register();
8146}
8147
8148Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8149 if (RetVT.SimpleTy != MVT::nxv2f16)
8150 return Register();
8151 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8152 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8153 }
8154 return Register();
8155}
8156
8157Register fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8158 if (RetVT.SimpleTy != MVT::nxv4f16)
8159 return Register();
8160 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8161 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8162 }
8163 return Register();
8164}
8165
8166Register fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8167 if (RetVT.SimpleTy != MVT::nxv8f16)
8168 return Register();
8169 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8170 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8171 }
8172 return Register();
8173}
8174
8175Register fastEmit_AArch64ISD_TRN1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8176 if (RetVT.SimpleTy != MVT::nxv2bf16)
8177 return Register();
8178 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8179 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8180 }
8181 return Register();
8182}
8183
8184Register fastEmit_AArch64ISD_TRN1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8185 if (RetVT.SimpleTy != MVT::nxv4bf16)
8186 return Register();
8187 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8188 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8189 }
8190 return Register();
8191}
8192
8193Register fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8194 if (RetVT.SimpleTy != MVT::nxv8bf16)
8195 return Register();
8196 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8197 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8198 }
8199 return Register();
8200}
8201
8202Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8203 if (RetVT.SimpleTy != MVT::nxv2f32)
8204 return Register();
8205 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8206 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8207 }
8208 return Register();
8209}
8210
8211Register fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8212 if (RetVT.SimpleTy != MVT::nxv4f32)
8213 return Register();
8214 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8215 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8216 }
8217 return Register();
8218}
8219
8220Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8221 if (RetVT.SimpleTy != MVT::nxv2f64)
8222 return Register();
8223 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8224 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8225 }
8226 return Register();
8227}
8228
8229Register fastEmit_AArch64ISD_TRN1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8230 switch (VT.SimpleTy) {
8231 case MVT::v8i8: return fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(RetVT, Op0, Op1);
8232 case MVT::v16i8: return fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(RetVT, Op0, Op1);
8233 case MVT::v4i16: return fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(RetVT, Op0, Op1);
8234 case MVT::v8i16: return fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(RetVT, Op0, Op1);
8235 case MVT::v2i32: return fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(RetVT, Op0, Op1);
8236 case MVT::v4i32: return fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(RetVT, Op0, Op1);
8237 case MVT::v2i64: return fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(RetVT, Op0, Op1);
8238 case MVT::v4f16: return fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(RetVT, Op0, Op1);
8239 case MVT::v8f16: return fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(RetVT, Op0, Op1);
8240 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(RetVT, Op0, Op1);
8241 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(RetVT, Op0, Op1);
8242 case MVT::v2f32: return fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(RetVT, Op0, Op1);
8243 case MVT::v4f32: return fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(RetVT, Op0, Op1);
8244 case MVT::v2f64: return fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(RetVT, Op0, Op1);
8245 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8246 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8247 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8248 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8249 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8250 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8251 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8252 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8253 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8254 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8255 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8256 case MVT::nxv2bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8257 case MVT::nxv4bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8258 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8259 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8260 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8261 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8262 default: return Register();
8263 }
8264}
8265
8266// FastEmit functions for AArch64ISD::TRN2.
8267
8268Register fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8269 if (RetVT.SimpleTy != MVT::v8i8)
8270 return Register();
8271 if ((Subtarget->isNeonAvailable())) {
8272 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8273 }
8274 return Register();
8275}
8276
8277Register fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8278 if (RetVT.SimpleTy != MVT::v16i8)
8279 return Register();
8280 if ((Subtarget->isNeonAvailable())) {
8281 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8282 }
8283 return Register();
8284}
8285
8286Register fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8287 if (RetVT.SimpleTy != MVT::v4i16)
8288 return Register();
8289 if ((Subtarget->isNeonAvailable())) {
8290 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8291 }
8292 return Register();
8293}
8294
8295Register fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8296 if (RetVT.SimpleTy != MVT::v8i16)
8297 return Register();
8298 if ((Subtarget->isNeonAvailable())) {
8299 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8300 }
8301 return Register();
8302}
8303
8304Register fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8305 if (RetVT.SimpleTy != MVT::v2i32)
8306 return Register();
8307 if ((Subtarget->isNeonAvailable())) {
8308 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8309 }
8310 return Register();
8311}
8312
8313Register fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8314 if (RetVT.SimpleTy != MVT::v4i32)
8315 return Register();
8316 if ((Subtarget->isNeonAvailable())) {
8317 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8318 }
8319 return Register();
8320}
8321
8322Register fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8323 if (RetVT.SimpleTy != MVT::v2i64)
8324 return Register();
8325 if ((Subtarget->isNeonAvailable())) {
8326 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8327 }
8328 return Register();
8329}
8330
8331Register fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8332 if (RetVT.SimpleTy != MVT::v4f16)
8333 return Register();
8334 if ((Subtarget->isNeonAvailable())) {
8335 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8336 }
8337 return Register();
8338}
8339
8340Register fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8341 if (RetVT.SimpleTy != MVT::v8f16)
8342 return Register();
8343 if ((Subtarget->isNeonAvailable())) {
8344 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8345 }
8346 return Register();
8347}
8348
8349Register fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8350 if (RetVT.SimpleTy != MVT::v4bf16)
8351 return Register();
8352 if ((Subtarget->isNeonAvailable())) {
8353 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8354 }
8355 return Register();
8356}
8357
8358Register fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8359 if (RetVT.SimpleTy != MVT::v8bf16)
8360 return Register();
8361 if ((Subtarget->isNeonAvailable())) {
8362 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8363 }
8364 return Register();
8365}
8366
8367Register fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8368 if (RetVT.SimpleTy != MVT::v2f32)
8369 return Register();
8370 if ((Subtarget->isNeonAvailable())) {
8371 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8372 }
8373 return Register();
8374}
8375
8376Register fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8377 if (RetVT.SimpleTy != MVT::v4f32)
8378 return Register();
8379 if ((Subtarget->isNeonAvailable())) {
8380 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8381 }
8382 return Register();
8383}
8384
8385Register fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8386 if (RetVT.SimpleTy != MVT::v2f64)
8387 return Register();
8388 if ((Subtarget->isNeonAvailable())) {
8389 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8390 }
8391 return Register();
8392}
8393
8394Register fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8395 if (RetVT.SimpleTy != MVT::nxv2i1)
8396 return Register();
8397 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8398 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8399 }
8400 return Register();
8401}
8402
8403Register fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8404 if (RetVT.SimpleTy != MVT::nxv4i1)
8405 return Register();
8406 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8407 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8408 }
8409 return Register();
8410}
8411
8412Register fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8413 if (RetVT.SimpleTy != MVT::nxv8i1)
8414 return Register();
8415 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8416 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8417 }
8418 return Register();
8419}
8420
8421Register fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8422 if (RetVT.SimpleTy != MVT::nxv16i1)
8423 return Register();
8424 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8425 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8426 }
8427 return Register();
8428}
8429
8430Register fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8431 if (RetVT.SimpleTy != MVT::nxv16i8)
8432 return Register();
8433 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8434 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8435 }
8436 return Register();
8437}
8438
8439Register fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8440 if (RetVT.SimpleTy != MVT::nxv8i16)
8441 return Register();
8442 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8443 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8444 }
8445 return Register();
8446}
8447
8448Register fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8449 if (RetVT.SimpleTy != MVT::nxv4i32)
8450 return Register();
8451 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8452 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8453 }
8454 return Register();
8455}
8456
8457Register fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8458 if (RetVT.SimpleTy != MVT::nxv2i64)
8459 return Register();
8460 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8461 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8462 }
8463 return Register();
8464}
8465
8466Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8467 if (RetVT.SimpleTy != MVT::nxv2f16)
8468 return Register();
8469 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8470 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8471 }
8472 return Register();
8473}
8474
8475Register fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8476 if (RetVT.SimpleTy != MVT::nxv4f16)
8477 return Register();
8478 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8479 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8480 }
8481 return Register();
8482}
8483
8484Register fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8485 if (RetVT.SimpleTy != MVT::nxv8f16)
8486 return Register();
8487 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8488 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8489 }
8490 return Register();
8491}
8492
8493Register fastEmit_AArch64ISD_TRN2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8494 if (RetVT.SimpleTy != MVT::nxv2bf16)
8495 return Register();
8496 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8497 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8498 }
8499 return Register();
8500}
8501
8502Register fastEmit_AArch64ISD_TRN2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8503 if (RetVT.SimpleTy != MVT::nxv4bf16)
8504 return Register();
8505 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8506 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8507 }
8508 return Register();
8509}
8510
8511Register fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8512 if (RetVT.SimpleTy != MVT::nxv8bf16)
8513 return Register();
8514 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8515 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8516 }
8517 return Register();
8518}
8519
8520Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8521 if (RetVT.SimpleTy != MVT::nxv2f32)
8522 return Register();
8523 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8524 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8525 }
8526 return Register();
8527}
8528
8529Register fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8530 if (RetVT.SimpleTy != MVT::nxv4f32)
8531 return Register();
8532 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8533 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8534 }
8535 return Register();
8536}
8537
8538Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8539 if (RetVT.SimpleTy != MVT::nxv2f64)
8540 return Register();
8541 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8542 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8543 }
8544 return Register();
8545}
8546
8547Register fastEmit_AArch64ISD_TRN2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8548 switch (VT.SimpleTy) {
8549 case MVT::v8i8: return fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(RetVT, Op0, Op1);
8550 case MVT::v16i8: return fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(RetVT, Op0, Op1);
8551 case MVT::v4i16: return fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(RetVT, Op0, Op1);
8552 case MVT::v8i16: return fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(RetVT, Op0, Op1);
8553 case MVT::v2i32: return fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(RetVT, Op0, Op1);
8554 case MVT::v4i32: return fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(RetVT, Op0, Op1);
8555 case MVT::v2i64: return fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(RetVT, Op0, Op1);
8556 case MVT::v4f16: return fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(RetVT, Op0, Op1);
8557 case MVT::v8f16: return fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(RetVT, Op0, Op1);
8558 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(RetVT, Op0, Op1);
8559 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(RetVT, Op0, Op1);
8560 case MVT::v2f32: return fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(RetVT, Op0, Op1);
8561 case MVT::v4f32: return fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(RetVT, Op0, Op1);
8562 case MVT::v2f64: return fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(RetVT, Op0, Op1);
8563 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8564 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8565 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8566 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8567 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8568 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8569 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8570 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8571 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8572 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8573 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8574 case MVT::nxv2bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8575 case MVT::nxv4bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8576 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8577 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8578 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8579 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8580 default: return Register();
8581 }
8582}
8583
8584// FastEmit functions for AArch64ISD::UMULL.
8585
8586Register fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8587 if (RetVT.SimpleTy != MVT::v8i16)
8588 return Register();
8589 if ((Subtarget->isNeonAvailable())) {
8590 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv8i8_v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8591 }
8592 return Register();
8593}
8594
8595Register fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8596 if (RetVT.SimpleTy != MVT::v4i32)
8597 return Register();
8598 if ((Subtarget->isNeonAvailable())) {
8599 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv4i16_v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8600 }
8601 return Register();
8602}
8603
8604Register fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8605 if (RetVT.SimpleTy != MVT::v2i64)
8606 return Register();
8607 if ((Subtarget->isNeonAvailable())) {
8608 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv2i32_v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8609 }
8610 return Register();
8611}
8612
8613Register fastEmit_AArch64ISD_UMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8614 switch (VT.SimpleTy) {
8615 case MVT::v8i8: return fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
8616 case MVT::v4i16: return fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(RetVT, Op0, Op1);
8617 case MVT::v2i32: return fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(RetVT, Op0, Op1);
8618 default: return Register();
8619 }
8620}
8621
8622// FastEmit functions for AArch64ISD::UQADD.
8623
8624Register fastEmit_AArch64ISD_UQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8625 if (RetVT.SimpleTy != MVT::f32)
8626 return Register();
8627 if ((Subtarget->isNeonAvailable())) {
8628 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8629 }
8630 return Register();
8631}
8632
8633Register fastEmit_AArch64ISD_UQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8634 if (RetVT.SimpleTy != MVT::f64)
8635 return Register();
8636 if ((Subtarget->isNeonAvailable())) {
8637 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8638 }
8639 return Register();
8640}
8641
8642Register fastEmit_AArch64ISD_UQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8643 switch (VT.SimpleTy) {
8644 case MVT::f32: return fastEmit_AArch64ISD_UQADD_MVT_f32_rr(RetVT, Op0, Op1);
8645 case MVT::f64: return fastEmit_AArch64ISD_UQADD_MVT_f64_rr(RetVT, Op0, Op1);
8646 default: return Register();
8647 }
8648}
8649
8650// FastEmit functions for AArch64ISD::UQRSHL.
8651
8652Register fastEmit_AArch64ISD_UQRSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8653 if (RetVT.SimpleTy != MVT::f32)
8654 return Register();
8655 if ((Subtarget->isNeonAvailable())) {
8656 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQRSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8657 }
8658 return Register();
8659}
8660
8661Register fastEmit_AArch64ISD_UQRSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8662 if (RetVT.SimpleTy != MVT::f64)
8663 return Register();
8664 if ((Subtarget->isNeonAvailable())) {
8665 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQRSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8666 }
8667 return Register();
8668}
8669
8670Register fastEmit_AArch64ISD_UQRSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8671 switch (VT.SimpleTy) {
8672 case MVT::f32: return fastEmit_AArch64ISD_UQRSHL_MVT_f32_rr(RetVT, Op0, Op1);
8673 case MVT::f64: return fastEmit_AArch64ISD_UQRSHL_MVT_f64_rr(RetVT, Op0, Op1);
8674 default: return Register();
8675 }
8676}
8677
8678// FastEmit functions for AArch64ISD::UQSHL.
8679
8680Register fastEmit_AArch64ISD_UQSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8681 if (RetVT.SimpleTy != MVT::f32)
8682 return Register();
8683 if ((Subtarget->isNeonAvailable())) {
8684 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8685 }
8686 return Register();
8687}
8688
8689Register fastEmit_AArch64ISD_UQSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8690 if (RetVT.SimpleTy != MVT::f64)
8691 return Register();
8692 if ((Subtarget->isNeonAvailable())) {
8693 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8694 }
8695 return Register();
8696}
8697
8698Register fastEmit_AArch64ISD_UQSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8699 switch (VT.SimpleTy) {
8700 case MVT::f32: return fastEmit_AArch64ISD_UQSHL_MVT_f32_rr(RetVT, Op0, Op1);
8701 case MVT::f64: return fastEmit_AArch64ISD_UQSHL_MVT_f64_rr(RetVT, Op0, Op1);
8702 default: return Register();
8703 }
8704}
8705
8706// FastEmit functions for AArch64ISD::UQSUB.
8707
8708Register fastEmit_AArch64ISD_UQSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8709 if (RetVT.SimpleTy != MVT::f32)
8710 return Register();
8711 if ((Subtarget->isNeonAvailable())) {
8712 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8713 }
8714 return Register();
8715}
8716
8717Register fastEmit_AArch64ISD_UQSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8718 if (RetVT.SimpleTy != MVT::f64)
8719 return Register();
8720 if ((Subtarget->isNeonAvailable())) {
8721 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8722 }
8723 return Register();
8724}
8725
8726Register fastEmit_AArch64ISD_UQSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8727 switch (VT.SimpleTy) {
8728 case MVT::f32: return fastEmit_AArch64ISD_UQSUB_MVT_f32_rr(RetVT, Op0, Op1);
8729 case MVT::f64: return fastEmit_AArch64ISD_UQSUB_MVT_f64_rr(RetVT, Op0, Op1);
8730 default: return Register();
8731 }
8732}
8733
8734// FastEmit functions for AArch64ISD::USQADD.
8735
8736Register fastEmit_AArch64ISD_USQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8737 if (RetVT.SimpleTy != MVT::f32)
8738 return Register();
8739 if ((Subtarget->isNeonAvailable())) {
8740 return fastEmitInst_rr(MachineInstOpcode: AArch64::USQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8741 }
8742 return Register();
8743}
8744
8745Register fastEmit_AArch64ISD_USQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8746 if (RetVT.SimpleTy != MVT::f64)
8747 return Register();
8748 if ((Subtarget->isNeonAvailable())) {
8749 return fastEmitInst_rr(MachineInstOpcode: AArch64::USQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8750 }
8751 return Register();
8752}
8753
8754Register fastEmit_AArch64ISD_USQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8755 switch (VT.SimpleTy) {
8756 case MVT::f32: return fastEmit_AArch64ISD_USQADD_MVT_f32_rr(RetVT, Op0, Op1);
8757 case MVT::f64: return fastEmit_AArch64ISD_USQADD_MVT_f64_rr(RetVT, Op0, Op1);
8758 default: return Register();
8759 }
8760}
8761
8762// FastEmit functions for AArch64ISD::UZP1.
8763
8764Register fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8765 if (RetVT.SimpleTy != MVT::v8i8)
8766 return Register();
8767 if ((Subtarget->isNeonAvailable())) {
8768 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8769 }
8770 return Register();
8771}
8772
8773Register fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8774 if (RetVT.SimpleTy != MVT::v16i8)
8775 return Register();
8776 if ((Subtarget->isNeonAvailable())) {
8777 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8778 }
8779 return Register();
8780}
8781
8782Register fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8783 if (RetVT.SimpleTy != MVT::v4i16)
8784 return Register();
8785 if ((Subtarget->isNeonAvailable())) {
8786 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8787 }
8788 return Register();
8789}
8790
8791Register fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8792 if (RetVT.SimpleTy != MVT::v8i16)
8793 return Register();
8794 if ((Subtarget->isNeonAvailable())) {
8795 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8796 }
8797 return Register();
8798}
8799
8800Register fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8801 if (RetVT.SimpleTy != MVT::v2i32)
8802 return Register();
8803 if ((Subtarget->isNeonAvailable())) {
8804 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8805 }
8806 return Register();
8807}
8808
8809Register fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8810 if (RetVT.SimpleTy != MVT::v4i32)
8811 return Register();
8812 if ((Subtarget->isNeonAvailable())) {
8813 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8814 }
8815 return Register();
8816}
8817
8818Register fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8819 if (RetVT.SimpleTy != MVT::v2i64)
8820 return Register();
8821 if ((Subtarget->isNeonAvailable())) {
8822 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8823 }
8824 return Register();
8825}
8826
8827Register fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8828 if (RetVT.SimpleTy != MVT::v4f16)
8829 return Register();
8830 if ((Subtarget->isNeonAvailable())) {
8831 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8832 }
8833 return Register();
8834}
8835
8836Register fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8837 if (RetVT.SimpleTy != MVT::v8f16)
8838 return Register();
8839 if ((Subtarget->isNeonAvailable())) {
8840 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8841 }
8842 return Register();
8843}
8844
8845Register fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8846 if (RetVT.SimpleTy != MVT::v4bf16)
8847 return Register();
8848 if ((Subtarget->isNeonAvailable())) {
8849 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8850 }
8851 return Register();
8852}
8853
8854Register fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8855 if (RetVT.SimpleTy != MVT::v8bf16)
8856 return Register();
8857 if ((Subtarget->isNeonAvailable())) {
8858 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8859 }
8860 return Register();
8861}
8862
8863Register fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8864 if (RetVT.SimpleTy != MVT::v2f32)
8865 return Register();
8866 if ((Subtarget->isNeonAvailable())) {
8867 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8868 }
8869 return Register();
8870}
8871
8872Register fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8873 if (RetVT.SimpleTy != MVT::v4f32)
8874 return Register();
8875 if ((Subtarget->isNeonAvailable())) {
8876 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8877 }
8878 return Register();
8879}
8880
8881Register fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8882 if (RetVT.SimpleTy != MVT::v2f64)
8883 return Register();
8884 if ((Subtarget->isNeonAvailable())) {
8885 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8886 }
8887 return Register();
8888}
8889
8890Register fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8891 if (RetVT.SimpleTy != MVT::nxv2i1)
8892 return Register();
8893 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8894 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8895 }
8896 return Register();
8897}
8898
8899Register fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8900 if (RetVT.SimpleTy != MVT::nxv4i1)
8901 return Register();
8902 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8903 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8904 }
8905 return Register();
8906}
8907
8908Register fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8909 if (RetVT.SimpleTy != MVT::nxv8i1)
8910 return Register();
8911 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8912 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8913 }
8914 return Register();
8915}
8916
8917Register fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8918 if (RetVT.SimpleTy != MVT::nxv16i1)
8919 return Register();
8920 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8921 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8922 }
8923 return Register();
8924}
8925
8926Register fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8927 if (RetVT.SimpleTy != MVT::nxv16i8)
8928 return Register();
8929 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8930 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8931 }
8932 return Register();
8933}
8934
8935Register fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8936 if (RetVT.SimpleTy != MVT::nxv8i16)
8937 return Register();
8938 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8939 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8940 }
8941 return Register();
8942}
8943
8944Register fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8945 if (RetVT.SimpleTy != MVT::nxv4i32)
8946 return Register();
8947 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8948 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8949 }
8950 return Register();
8951}
8952
8953Register fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8954 if (RetVT.SimpleTy != MVT::nxv2i64)
8955 return Register();
8956 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8957 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8958 }
8959 return Register();
8960}
8961
8962Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8963 if (RetVT.SimpleTy != MVT::nxv2f16)
8964 return Register();
8965 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8966 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8967 }
8968 return Register();
8969}
8970
8971Register fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8972 if (RetVT.SimpleTy != MVT::nxv4f16)
8973 return Register();
8974 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8975 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8976 }
8977 return Register();
8978}
8979
8980Register fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8981 if (RetVT.SimpleTy != MVT::nxv8f16)
8982 return Register();
8983 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8984 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8985 }
8986 return Register();
8987}
8988
8989Register fastEmit_AArch64ISD_UZP1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8990 if (RetVT.SimpleTy != MVT::nxv2bf16)
8991 return Register();
8992 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8993 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8994 }
8995 return Register();
8996}
8997
8998Register fastEmit_AArch64ISD_UZP1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8999 if (RetVT.SimpleTy != MVT::nxv4bf16)
9000 return Register();
9001 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9002 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9003 }
9004 return Register();
9005}
9006
9007Register fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9008 if (RetVT.SimpleTy != MVT::nxv8bf16)
9009 return Register();
9010 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9011 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9012 }
9013 return Register();
9014}
9015
9016Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9017 if (RetVT.SimpleTy != MVT::nxv2f32)
9018 return Register();
9019 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9020 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9021 }
9022 return Register();
9023}
9024
9025Register fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9026 if (RetVT.SimpleTy != MVT::nxv4f32)
9027 return Register();
9028 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9029 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9030 }
9031 return Register();
9032}
9033
9034Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9035 if (RetVT.SimpleTy != MVT::nxv2f64)
9036 return Register();
9037 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9038 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9039 }
9040 return Register();
9041}
9042
9043Register fastEmit_AArch64ISD_UZP1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9044 switch (VT.SimpleTy) {
9045 case MVT::v8i8: return fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(RetVT, Op0, Op1);
9046 case MVT::v16i8: return fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(RetVT, Op0, Op1);
9047 case MVT::v4i16: return fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(RetVT, Op0, Op1);
9048 case MVT::v8i16: return fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(RetVT, Op0, Op1);
9049 case MVT::v2i32: return fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(RetVT, Op0, Op1);
9050 case MVT::v4i32: return fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(RetVT, Op0, Op1);
9051 case MVT::v2i64: return fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(RetVT, Op0, Op1);
9052 case MVT::v4f16: return fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(RetVT, Op0, Op1);
9053 case MVT::v8f16: return fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(RetVT, Op0, Op1);
9054 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(RetVT, Op0, Op1);
9055 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(RetVT, Op0, Op1);
9056 case MVT::v2f32: return fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(RetVT, Op0, Op1);
9057 case MVT::v4f32: return fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(RetVT, Op0, Op1);
9058 case MVT::v2f64: return fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(RetVT, Op0, Op1);
9059 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9060 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9061 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9062 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9063 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9064 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9065 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9066 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9067 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9068 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9069 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9070 case MVT::nxv2bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9071 case MVT::nxv4bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9072 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9073 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9074 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9075 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9076 default: return Register();
9077 }
9078}
9079
9080// FastEmit functions for AArch64ISD::UZP2.
9081
9082Register fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9083 if (RetVT.SimpleTy != MVT::v8i8)
9084 return Register();
9085 if ((Subtarget->isNeonAvailable())) {
9086 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9087 }
9088 return Register();
9089}
9090
9091Register fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9092 if (RetVT.SimpleTy != MVT::v16i8)
9093 return Register();
9094 if ((Subtarget->isNeonAvailable())) {
9095 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9096 }
9097 return Register();
9098}
9099
9100Register fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9101 if (RetVT.SimpleTy != MVT::v4i16)
9102 return Register();
9103 if ((Subtarget->isNeonAvailable())) {
9104 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9105 }
9106 return Register();
9107}
9108
9109Register fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9110 if (RetVT.SimpleTy != MVT::v8i16)
9111 return Register();
9112 if ((Subtarget->isNeonAvailable())) {
9113 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9114 }
9115 return Register();
9116}
9117
9118Register fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9119 if (RetVT.SimpleTy != MVT::v2i32)
9120 return Register();
9121 if ((Subtarget->isNeonAvailable())) {
9122 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9123 }
9124 return Register();
9125}
9126
9127Register fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9128 if (RetVT.SimpleTy != MVT::v4i32)
9129 return Register();
9130 if ((Subtarget->isNeonAvailable())) {
9131 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9132 }
9133 return Register();
9134}
9135
9136Register fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9137 if (RetVT.SimpleTy != MVT::v2i64)
9138 return Register();
9139 if ((Subtarget->isNeonAvailable())) {
9140 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9141 }
9142 return Register();
9143}
9144
9145Register fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9146 if (RetVT.SimpleTy != MVT::v4f16)
9147 return Register();
9148 if ((Subtarget->isNeonAvailable())) {
9149 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9150 }
9151 return Register();
9152}
9153
9154Register fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9155 if (RetVT.SimpleTy != MVT::v8f16)
9156 return Register();
9157 if ((Subtarget->isNeonAvailable())) {
9158 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9159 }
9160 return Register();
9161}
9162
9163Register fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9164 if (RetVT.SimpleTy != MVT::v4bf16)
9165 return Register();
9166 if ((Subtarget->isNeonAvailable())) {
9167 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9168 }
9169 return Register();
9170}
9171
9172Register fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9173 if (RetVT.SimpleTy != MVT::v8bf16)
9174 return Register();
9175 if ((Subtarget->isNeonAvailable())) {
9176 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9177 }
9178 return Register();
9179}
9180
9181Register fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9182 if (RetVT.SimpleTy != MVT::v2f32)
9183 return Register();
9184 if ((Subtarget->isNeonAvailable())) {
9185 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9186 }
9187 return Register();
9188}
9189
9190Register fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9191 if (RetVT.SimpleTy != MVT::v4f32)
9192 return Register();
9193 if ((Subtarget->isNeonAvailable())) {
9194 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9195 }
9196 return Register();
9197}
9198
9199Register fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9200 if (RetVT.SimpleTy != MVT::v2f64)
9201 return Register();
9202 if ((Subtarget->isNeonAvailable())) {
9203 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9204 }
9205 return Register();
9206}
9207
9208Register fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9209 if (RetVT.SimpleTy != MVT::nxv2i1)
9210 return Register();
9211 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9212 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9213 }
9214 return Register();
9215}
9216
9217Register fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9218 if (RetVT.SimpleTy != MVT::nxv4i1)
9219 return Register();
9220 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9221 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9222 }
9223 return Register();
9224}
9225
9226Register fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9227 if (RetVT.SimpleTy != MVT::nxv8i1)
9228 return Register();
9229 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9230 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9231 }
9232 return Register();
9233}
9234
9235Register fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9236 if (RetVT.SimpleTy != MVT::nxv16i1)
9237 return Register();
9238 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9239 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9240 }
9241 return Register();
9242}
9243
9244Register fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9245 if (RetVT.SimpleTy != MVT::nxv16i8)
9246 return Register();
9247 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9248 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9249 }
9250 return Register();
9251}
9252
9253Register fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9254 if (RetVT.SimpleTy != MVT::nxv8i16)
9255 return Register();
9256 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9257 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9258 }
9259 return Register();
9260}
9261
9262Register fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9263 if (RetVT.SimpleTy != MVT::nxv4i32)
9264 return Register();
9265 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9266 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9267 }
9268 return Register();
9269}
9270
9271Register fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9272 if (RetVT.SimpleTy != MVT::nxv2i64)
9273 return Register();
9274 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9275 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9276 }
9277 return Register();
9278}
9279
9280Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9281 if (RetVT.SimpleTy != MVT::nxv2f16)
9282 return Register();
9283 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9284 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9285 }
9286 return Register();
9287}
9288
9289Register fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9290 if (RetVT.SimpleTy != MVT::nxv4f16)
9291 return Register();
9292 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9293 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9294 }
9295 return Register();
9296}
9297
9298Register fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9299 if (RetVT.SimpleTy != MVT::nxv8f16)
9300 return Register();
9301 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9302 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9303 }
9304 return Register();
9305}
9306
9307Register fastEmit_AArch64ISD_UZP2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9308 if (RetVT.SimpleTy != MVT::nxv2bf16)
9309 return Register();
9310 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9311 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9312 }
9313 return Register();
9314}
9315
9316Register fastEmit_AArch64ISD_UZP2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9317 if (RetVT.SimpleTy != MVT::nxv4bf16)
9318 return Register();
9319 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9320 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9321 }
9322 return Register();
9323}
9324
9325Register fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9326 if (RetVT.SimpleTy != MVT::nxv8bf16)
9327 return Register();
9328 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9329 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9330 }
9331 return Register();
9332}
9333
9334Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9335 if (RetVT.SimpleTy != MVT::nxv2f32)
9336 return Register();
9337 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9338 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9339 }
9340 return Register();
9341}
9342
9343Register fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9344 if (RetVT.SimpleTy != MVT::nxv4f32)
9345 return Register();
9346 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9347 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9348 }
9349 return Register();
9350}
9351
9352Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9353 if (RetVT.SimpleTy != MVT::nxv2f64)
9354 return Register();
9355 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9356 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9357 }
9358 return Register();
9359}
9360
9361Register fastEmit_AArch64ISD_UZP2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9362 switch (VT.SimpleTy) {
9363 case MVT::v8i8: return fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(RetVT, Op0, Op1);
9364 case MVT::v16i8: return fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(RetVT, Op0, Op1);
9365 case MVT::v4i16: return fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(RetVT, Op0, Op1);
9366 case MVT::v8i16: return fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(RetVT, Op0, Op1);
9367 case MVT::v2i32: return fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(RetVT, Op0, Op1);
9368 case MVT::v4i32: return fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(RetVT, Op0, Op1);
9369 case MVT::v2i64: return fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(RetVT, Op0, Op1);
9370 case MVT::v4f16: return fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(RetVT, Op0, Op1);
9371 case MVT::v8f16: return fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(RetVT, Op0, Op1);
9372 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(RetVT, Op0, Op1);
9373 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(RetVT, Op0, Op1);
9374 case MVT::v2f32: return fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(RetVT, Op0, Op1);
9375 case MVT::v4f32: return fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(RetVT, Op0, Op1);
9376 case MVT::v2f64: return fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(RetVT, Op0, Op1);
9377 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9378 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9379 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9380 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9381 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9382 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9383 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9384 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9385 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9386 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9387 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9388 case MVT::nxv2bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9389 case MVT::nxv4bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9390 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9391 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9392 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9393 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9394 default: return Register();
9395 }
9396}
9397
9398// FastEmit functions for AArch64ISD::ZIP1.
9399
9400Register fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9401 if (RetVT.SimpleTy != MVT::v8i8)
9402 return Register();
9403 if ((Subtarget->isNeonAvailable())) {
9404 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9405 }
9406 return Register();
9407}
9408
9409Register fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9410 if (RetVT.SimpleTy != MVT::v16i8)
9411 return Register();
9412 if ((Subtarget->isNeonAvailable())) {
9413 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9414 }
9415 return Register();
9416}
9417
9418Register fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9419 if (RetVT.SimpleTy != MVT::v4i16)
9420 return Register();
9421 if ((Subtarget->isNeonAvailable())) {
9422 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9423 }
9424 return Register();
9425}
9426
9427Register fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9428 if (RetVT.SimpleTy != MVT::v8i16)
9429 return Register();
9430 if ((Subtarget->isNeonAvailable())) {
9431 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9432 }
9433 return Register();
9434}
9435
9436Register fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9437 if (RetVT.SimpleTy != MVT::v2i32)
9438 return Register();
9439 if ((Subtarget->isNeonAvailable())) {
9440 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9441 }
9442 return Register();
9443}
9444
9445Register fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9446 if (RetVT.SimpleTy != MVT::v4i32)
9447 return Register();
9448 if ((Subtarget->isNeonAvailable())) {
9449 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9450 }
9451 return Register();
9452}
9453
9454Register fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9455 if (RetVT.SimpleTy != MVT::v2i64)
9456 return Register();
9457 if ((Subtarget->isNeonAvailable())) {
9458 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9459 }
9460 return Register();
9461}
9462
9463Register fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9464 if (RetVT.SimpleTy != MVT::v4f16)
9465 return Register();
9466 if ((Subtarget->isNeonAvailable())) {
9467 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9468 }
9469 return Register();
9470}
9471
9472Register fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9473 if (RetVT.SimpleTy != MVT::v8f16)
9474 return Register();
9475 if ((Subtarget->isNeonAvailable())) {
9476 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9477 }
9478 return Register();
9479}
9480
9481Register fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9482 if (RetVT.SimpleTy != MVT::v4bf16)
9483 return Register();
9484 if ((Subtarget->isNeonAvailable())) {
9485 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9486 }
9487 return Register();
9488}
9489
9490Register fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9491 if (RetVT.SimpleTy != MVT::v8bf16)
9492 return Register();
9493 if ((Subtarget->isNeonAvailable())) {
9494 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9495 }
9496 return Register();
9497}
9498
9499Register fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9500 if (RetVT.SimpleTy != MVT::v2f32)
9501 return Register();
9502 if ((Subtarget->isNeonAvailable())) {
9503 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9504 }
9505 return Register();
9506}
9507
9508Register fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9509 if (RetVT.SimpleTy != MVT::v4f32)
9510 return Register();
9511 if ((Subtarget->isNeonAvailable())) {
9512 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9513 }
9514 return Register();
9515}
9516
9517Register fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9518 if (RetVT.SimpleTy != MVT::v2f64)
9519 return Register();
9520 if ((Subtarget->isNeonAvailable())) {
9521 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9522 }
9523 return Register();
9524}
9525
9526Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9527 if (RetVT.SimpleTy != MVT::nxv2i1)
9528 return Register();
9529 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9530 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9531 }
9532 return Register();
9533}
9534
9535Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9536 if (RetVT.SimpleTy != MVT::nxv4i1)
9537 return Register();
9538 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9539 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9540 }
9541 return Register();
9542}
9543
9544Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9545 if (RetVT.SimpleTy != MVT::nxv8i1)
9546 return Register();
9547 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9548 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9549 }
9550 return Register();
9551}
9552
9553Register fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9554 if (RetVT.SimpleTy != MVT::nxv16i1)
9555 return Register();
9556 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9557 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9558 }
9559 return Register();
9560}
9561
9562Register fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9563 if (RetVT.SimpleTy != MVT::nxv16i8)
9564 return Register();
9565 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9566 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9567 }
9568 return Register();
9569}
9570
9571Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9572 if (RetVT.SimpleTy != MVT::nxv8i16)
9573 return Register();
9574 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9575 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9576 }
9577 return Register();
9578}
9579
9580Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9581 if (RetVT.SimpleTy != MVT::nxv4i32)
9582 return Register();
9583 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9584 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9585 }
9586 return Register();
9587}
9588
9589Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9590 if (RetVT.SimpleTy != MVT::nxv2i64)
9591 return Register();
9592 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9593 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9594 }
9595 return Register();
9596}
9597
9598Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9599 if (RetVT.SimpleTy != MVT::nxv2f16)
9600 return Register();
9601 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9602 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9603 }
9604 return Register();
9605}
9606
9607Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9608 if (RetVT.SimpleTy != MVT::nxv4f16)
9609 return Register();
9610 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9611 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9612 }
9613 return Register();
9614}
9615
9616Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9617 if (RetVT.SimpleTy != MVT::nxv8f16)
9618 return Register();
9619 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9620 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9621 }
9622 return Register();
9623}
9624
9625Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9626 if (RetVT.SimpleTy != MVT::nxv2bf16)
9627 return Register();
9628 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9629 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9630 }
9631 return Register();
9632}
9633
9634Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9635 if (RetVT.SimpleTy != MVT::nxv4bf16)
9636 return Register();
9637 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9638 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9639 }
9640 return Register();
9641}
9642
9643Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9644 if (RetVT.SimpleTy != MVT::nxv8bf16)
9645 return Register();
9646 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9647 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9648 }
9649 return Register();
9650}
9651
9652Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9653 if (RetVT.SimpleTy != MVT::nxv2f32)
9654 return Register();
9655 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9656 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9657 }
9658 return Register();
9659}
9660
9661Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9662 if (RetVT.SimpleTy != MVT::nxv4f32)
9663 return Register();
9664 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9665 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9666 }
9667 return Register();
9668}
9669
9670Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9671 if (RetVT.SimpleTy != MVT::nxv2f64)
9672 return Register();
9673 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9674 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9675 }
9676 return Register();
9677}
9678
9679Register fastEmit_AArch64ISD_ZIP1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9680 switch (VT.SimpleTy) {
9681 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(RetVT, Op0, Op1);
9682 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(RetVT, Op0, Op1);
9683 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(RetVT, Op0, Op1);
9684 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(RetVT, Op0, Op1);
9685 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(RetVT, Op0, Op1);
9686 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(RetVT, Op0, Op1);
9687 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(RetVT, Op0, Op1);
9688 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(RetVT, Op0, Op1);
9689 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(RetVT, Op0, Op1);
9690 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(RetVT, Op0, Op1);
9691 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(RetVT, Op0, Op1);
9692 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(RetVT, Op0, Op1);
9693 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(RetVT, Op0, Op1);
9694 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(RetVT, Op0, Op1);
9695 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9696 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9697 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9698 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9699 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9700 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9701 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9702 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9703 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9704 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9705 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9706 case MVT::nxv2bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9707 case MVT::nxv4bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9708 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9709 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9710 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9711 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9712 default: return Register();
9713 }
9714}
9715
9716// FastEmit functions for AArch64ISD::ZIP2.
9717
9718Register fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9719 if (RetVT.SimpleTy != MVT::v8i8)
9720 return Register();
9721 if ((Subtarget->isNeonAvailable())) {
9722 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9723 }
9724 return Register();
9725}
9726
9727Register fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9728 if (RetVT.SimpleTy != MVT::v16i8)
9729 return Register();
9730 if ((Subtarget->isNeonAvailable())) {
9731 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9732 }
9733 return Register();
9734}
9735
9736Register fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9737 if (RetVT.SimpleTy != MVT::v4i16)
9738 return Register();
9739 if ((Subtarget->isNeonAvailable())) {
9740 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9741 }
9742 return Register();
9743}
9744
9745Register fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9746 if (RetVT.SimpleTy != MVT::v8i16)
9747 return Register();
9748 if ((Subtarget->isNeonAvailable())) {
9749 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9750 }
9751 return Register();
9752}
9753
9754Register fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9755 if (RetVT.SimpleTy != MVT::v2i32)
9756 return Register();
9757 if ((Subtarget->isNeonAvailable())) {
9758 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9759 }
9760 return Register();
9761}
9762
9763Register fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9764 if (RetVT.SimpleTy != MVT::v4i32)
9765 return Register();
9766 if ((Subtarget->isNeonAvailable())) {
9767 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9768 }
9769 return Register();
9770}
9771
9772Register fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9773 if (RetVT.SimpleTy != MVT::v2i64)
9774 return Register();
9775 if ((Subtarget->isNeonAvailable())) {
9776 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9777 }
9778 return Register();
9779}
9780
9781Register fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9782 if (RetVT.SimpleTy != MVT::v4f16)
9783 return Register();
9784 if ((Subtarget->isNeonAvailable())) {
9785 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9786 }
9787 return Register();
9788}
9789
9790Register fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9791 if (RetVT.SimpleTy != MVT::v8f16)
9792 return Register();
9793 if ((Subtarget->isNeonAvailable())) {
9794 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9795 }
9796 return Register();
9797}
9798
9799Register fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9800 if (RetVT.SimpleTy != MVT::v4bf16)
9801 return Register();
9802 if ((Subtarget->isNeonAvailable())) {
9803 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9804 }
9805 return Register();
9806}
9807
9808Register fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9809 if (RetVT.SimpleTy != MVT::v8bf16)
9810 return Register();
9811 if ((Subtarget->isNeonAvailable())) {
9812 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9813 }
9814 return Register();
9815}
9816
9817Register fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9818 if (RetVT.SimpleTy != MVT::v2f32)
9819 return Register();
9820 if ((Subtarget->isNeonAvailable())) {
9821 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9822 }
9823 return Register();
9824}
9825
9826Register fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9827 if (RetVT.SimpleTy != MVT::v4f32)
9828 return Register();
9829 if ((Subtarget->isNeonAvailable())) {
9830 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9831 }
9832 return Register();
9833}
9834
9835Register fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9836 if (RetVT.SimpleTy != MVT::v2f64)
9837 return Register();
9838 if ((Subtarget->isNeonAvailable())) {
9839 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9840 }
9841 return Register();
9842}
9843
9844Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9845 if (RetVT.SimpleTy != MVT::nxv2i1)
9846 return Register();
9847 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9848 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9849 }
9850 return Register();
9851}
9852
9853Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9854 if (RetVT.SimpleTy != MVT::nxv4i1)
9855 return Register();
9856 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9857 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9858 }
9859 return Register();
9860}
9861
9862Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9863 if (RetVT.SimpleTy != MVT::nxv8i1)
9864 return Register();
9865 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9866 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9867 }
9868 return Register();
9869}
9870
9871Register fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9872 if (RetVT.SimpleTy != MVT::nxv16i1)
9873 return Register();
9874 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9875 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9876 }
9877 return Register();
9878}
9879
9880Register fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9881 if (RetVT.SimpleTy != MVT::nxv16i8)
9882 return Register();
9883 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9884 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9885 }
9886 return Register();
9887}
9888
9889Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9890 if (RetVT.SimpleTy != MVT::nxv8i16)
9891 return Register();
9892 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9893 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9894 }
9895 return Register();
9896}
9897
9898Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9899 if (RetVT.SimpleTy != MVT::nxv4i32)
9900 return Register();
9901 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9902 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9903 }
9904 return Register();
9905}
9906
9907Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9908 if (RetVT.SimpleTy != MVT::nxv2i64)
9909 return Register();
9910 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9911 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9912 }
9913 return Register();
9914}
9915
9916Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9917 if (RetVT.SimpleTy != MVT::nxv2f16)
9918 return Register();
9919 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9920 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9921 }
9922 return Register();
9923}
9924
9925Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9926 if (RetVT.SimpleTy != MVT::nxv4f16)
9927 return Register();
9928 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9929 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9930 }
9931 return Register();
9932}
9933
9934Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9935 if (RetVT.SimpleTy != MVT::nxv8f16)
9936 return Register();
9937 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9938 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9939 }
9940 return Register();
9941}
9942
9943Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9944 if (RetVT.SimpleTy != MVT::nxv2bf16)
9945 return Register();
9946 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9947 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9948 }
9949 return Register();
9950}
9951
9952Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9953 if (RetVT.SimpleTy != MVT::nxv4bf16)
9954 return Register();
9955 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9956 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9957 }
9958 return Register();
9959}
9960
9961Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9962 if (RetVT.SimpleTy != MVT::nxv8bf16)
9963 return Register();
9964 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9965 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9966 }
9967 return Register();
9968}
9969
9970Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9971 if (RetVT.SimpleTy != MVT::nxv2f32)
9972 return Register();
9973 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9974 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9975 }
9976 return Register();
9977}
9978
9979Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9980 if (RetVT.SimpleTy != MVT::nxv4f32)
9981 return Register();
9982 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9983 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9984 }
9985 return Register();
9986}
9987
9988Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9989 if (RetVT.SimpleTy != MVT::nxv2f64)
9990 return Register();
9991 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9992 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9993 }
9994 return Register();
9995}
9996
9997Register fastEmit_AArch64ISD_ZIP2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9998 switch (VT.SimpleTy) {
9999 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(RetVT, Op0, Op1);
10000 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(RetVT, Op0, Op1);
10001 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(RetVT, Op0, Op1);
10002 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(RetVT, Op0, Op1);
10003 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(RetVT, Op0, Op1);
10004 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(RetVT, Op0, Op1);
10005 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(RetVT, Op0, Op1);
10006 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(RetVT, Op0, Op1);
10007 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(RetVT, Op0, Op1);
10008 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(RetVT, Op0, Op1);
10009 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(RetVT, Op0, Op1);
10010 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(RetVT, Op0, Op1);
10011 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(RetVT, Op0, Op1);
10012 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(RetVT, Op0, Op1);
10013 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
10014 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
10015 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
10016 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
10017 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10018 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10019 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10020 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10021 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
10022 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
10023 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
10024 case MVT::nxv2bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
10025 case MVT::nxv4bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
10026 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
10027 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
10028 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
10029 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
10030 default: return Register();
10031 }
10032}
10033
10034// FastEmit functions for ISD::ABDS.
10035
10036Register fastEmit_ISD_ABDS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10037 if (RetVT.SimpleTy != MVT::v8i8)
10038 return Register();
10039 if ((Subtarget->isNeonAvailable())) {
10040 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10041 }
10042 return Register();
10043}
10044
10045Register fastEmit_ISD_ABDS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10046 if (RetVT.SimpleTy != MVT::v16i8)
10047 return Register();
10048 if ((Subtarget->isNeonAvailable())) {
10049 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10050 }
10051 return Register();
10052}
10053
10054Register fastEmit_ISD_ABDS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10055 if (RetVT.SimpleTy != MVT::v4i16)
10056 return Register();
10057 if ((Subtarget->isNeonAvailable())) {
10058 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10059 }
10060 return Register();
10061}
10062
10063Register fastEmit_ISD_ABDS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10064 if (RetVT.SimpleTy != MVT::v8i16)
10065 return Register();
10066 if ((Subtarget->isNeonAvailable())) {
10067 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10068 }
10069 return Register();
10070}
10071
10072Register fastEmit_ISD_ABDS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10073 if (RetVT.SimpleTy != MVT::v2i32)
10074 return Register();
10075 if ((Subtarget->isNeonAvailable())) {
10076 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10077 }
10078 return Register();
10079}
10080
10081Register fastEmit_ISD_ABDS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10082 if (RetVT.SimpleTy != MVT::v4i32)
10083 return Register();
10084 if ((Subtarget->isNeonAvailable())) {
10085 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10086 }
10087 return Register();
10088}
10089
10090Register fastEmit_ISD_ABDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10091 switch (VT.SimpleTy) {
10092 case MVT::v8i8: return fastEmit_ISD_ABDS_MVT_v8i8_rr(RetVT, Op0, Op1);
10093 case MVT::v16i8: return fastEmit_ISD_ABDS_MVT_v16i8_rr(RetVT, Op0, Op1);
10094 case MVT::v4i16: return fastEmit_ISD_ABDS_MVT_v4i16_rr(RetVT, Op0, Op1);
10095 case MVT::v8i16: return fastEmit_ISD_ABDS_MVT_v8i16_rr(RetVT, Op0, Op1);
10096 case MVT::v2i32: return fastEmit_ISD_ABDS_MVT_v2i32_rr(RetVT, Op0, Op1);
10097 case MVT::v4i32: return fastEmit_ISD_ABDS_MVT_v4i32_rr(RetVT, Op0, Op1);
10098 default: return Register();
10099 }
10100}
10101
10102// FastEmit functions for ISD::ABDU.
10103
10104Register fastEmit_ISD_ABDU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10105 if (RetVT.SimpleTy != MVT::v8i8)
10106 return Register();
10107 if ((Subtarget->isNeonAvailable())) {
10108 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10109 }
10110 return Register();
10111}
10112
10113Register fastEmit_ISD_ABDU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10114 if (RetVT.SimpleTy != MVT::v16i8)
10115 return Register();
10116 if ((Subtarget->isNeonAvailable())) {
10117 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10118 }
10119 return Register();
10120}
10121
10122Register fastEmit_ISD_ABDU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10123 if (RetVT.SimpleTy != MVT::v4i16)
10124 return Register();
10125 if ((Subtarget->isNeonAvailable())) {
10126 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10127 }
10128 return Register();
10129}
10130
10131Register fastEmit_ISD_ABDU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10132 if (RetVT.SimpleTy != MVT::v8i16)
10133 return Register();
10134 if ((Subtarget->isNeonAvailable())) {
10135 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10136 }
10137 return Register();
10138}
10139
10140Register fastEmit_ISD_ABDU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10141 if (RetVT.SimpleTy != MVT::v2i32)
10142 return Register();
10143 if ((Subtarget->isNeonAvailable())) {
10144 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10145 }
10146 return Register();
10147}
10148
10149Register fastEmit_ISD_ABDU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10150 if (RetVT.SimpleTy != MVT::v4i32)
10151 return Register();
10152 if ((Subtarget->isNeonAvailable())) {
10153 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10154 }
10155 return Register();
10156}
10157
10158Register fastEmit_ISD_ABDU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10159 switch (VT.SimpleTy) {
10160 case MVT::v8i8: return fastEmit_ISD_ABDU_MVT_v8i8_rr(RetVT, Op0, Op1);
10161 case MVT::v16i8: return fastEmit_ISD_ABDU_MVT_v16i8_rr(RetVT, Op0, Op1);
10162 case MVT::v4i16: return fastEmit_ISD_ABDU_MVT_v4i16_rr(RetVT, Op0, Op1);
10163 case MVT::v8i16: return fastEmit_ISD_ABDU_MVT_v8i16_rr(RetVT, Op0, Op1);
10164 case MVT::v2i32: return fastEmit_ISD_ABDU_MVT_v2i32_rr(RetVT, Op0, Op1);
10165 case MVT::v4i32: return fastEmit_ISD_ABDU_MVT_v4i32_rr(RetVT, Op0, Op1);
10166 default: return Register();
10167 }
10168}
10169
10170// FastEmit functions for ISD::ADD.
10171
10172Register fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10173 if (RetVT.SimpleTy != MVT::i32)
10174 return Register();
10175 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
10176}
10177
10178Register fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10179 if (RetVT.SimpleTy != MVT::i64)
10180 return Register();
10181 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
10182}
10183
10184Register fastEmit_ISD_ADD_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10185 if (RetVT.SimpleTy != MVT::v8i8)
10186 return Register();
10187 if ((Subtarget->isNeonAvailable())) {
10188 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10189 }
10190 return Register();
10191}
10192
10193Register fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10194 if (RetVT.SimpleTy != MVT::v16i8)
10195 return Register();
10196 if ((Subtarget->isNeonAvailable())) {
10197 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10198 }
10199 return Register();
10200}
10201
10202Register fastEmit_ISD_ADD_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10203 if (RetVT.SimpleTy != MVT::v4i16)
10204 return Register();
10205 if ((Subtarget->isNeonAvailable())) {
10206 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10207 }
10208 return Register();
10209}
10210
10211Register fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10212 if (RetVT.SimpleTy != MVT::v8i16)
10213 return Register();
10214 if ((Subtarget->isNeonAvailable())) {
10215 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10216 }
10217 return Register();
10218}
10219
10220Register fastEmit_ISD_ADD_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10221 if (RetVT.SimpleTy != MVT::v2i32)
10222 return Register();
10223 if ((Subtarget->isNeonAvailable())) {
10224 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10225 }
10226 return Register();
10227}
10228
10229Register fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10230 if (RetVT.SimpleTy != MVT::v4i32)
10231 return Register();
10232 if ((Subtarget->isNeonAvailable())) {
10233 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10234 }
10235 return Register();
10236}
10237
10238Register fastEmit_ISD_ADD_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
10239 if (RetVT.SimpleTy != MVT::v1i64)
10240 return Register();
10241 if ((Subtarget->isNeonAvailable())) {
10242 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
10243 }
10244 return Register();
10245}
10246
10247Register fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10248 if (RetVT.SimpleTy != MVT::v2i64)
10249 return Register();
10250 if ((Subtarget->isNeonAvailable())) {
10251 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10252 }
10253 return Register();
10254}
10255
10256Register fastEmit_ISD_ADD_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10257 if (RetVT.SimpleTy != MVT::nxv16i8)
10258 return Register();
10259 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10260 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
10261 }
10262 return Register();
10263}
10264
10265Register fastEmit_ISD_ADD_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10266 if (RetVT.SimpleTy != MVT::nxv8i16)
10267 return Register();
10268 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10269 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10270 }
10271 return Register();
10272}
10273
10274Register fastEmit_ISD_ADD_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10275 if (RetVT.SimpleTy != MVT::nxv4i32)
10276 return Register();
10277 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10278 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10279 }
10280 return Register();
10281}
10282
10283Register fastEmit_ISD_ADD_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10284 if (RetVT.SimpleTy != MVT::nxv2i64)
10285 return Register();
10286 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10287 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10288 }
10289 return Register();
10290}
10291
10292Register fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10293 switch (VT.SimpleTy) {
10294 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
10295 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op1);
10296 case MVT::v8i8: return fastEmit_ISD_ADD_MVT_v8i8_rr(RetVT, Op0, Op1);
10297 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
10298 case MVT::v4i16: return fastEmit_ISD_ADD_MVT_v4i16_rr(RetVT, Op0, Op1);
10299 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
10300 case MVT::v2i32: return fastEmit_ISD_ADD_MVT_v2i32_rr(RetVT, Op0, Op1);
10301 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
10302 case MVT::v1i64: return fastEmit_ISD_ADD_MVT_v1i64_rr(RetVT, Op0, Op1);
10303 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
10304 case MVT::nxv16i8: return fastEmit_ISD_ADD_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10305 case MVT::nxv8i16: return fastEmit_ISD_ADD_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10306 case MVT::nxv4i32: return fastEmit_ISD_ADD_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10307 case MVT::nxv2i64: return fastEmit_ISD_ADD_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10308 default: return Register();
10309 }
10310}
10311
10312// FastEmit functions for ISD::AND.
10313
10314Register fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10315 if (RetVT.SimpleTy != MVT::i32)
10316 return Register();
10317 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
10318}
10319
10320Register fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10321 if (RetVT.SimpleTy != MVT::i64)
10322 return Register();
10323 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
10324}
10325
10326Register fastEmit_ISD_AND_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10327 if (RetVT.SimpleTy != MVT::v8i8)
10328 return Register();
10329 if ((Subtarget->isNeonAvailable())) {
10330 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10331 }
10332 return Register();
10333}
10334
10335Register fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10336 if (RetVT.SimpleTy != MVT::v16i8)
10337 return Register();
10338 if ((Subtarget->isNeonAvailable())) {
10339 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10340 }
10341 return Register();
10342}
10343
10344Register fastEmit_ISD_AND_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10345 if (RetVT.SimpleTy != MVT::v4i16)
10346 return Register();
10347 if ((Subtarget->isNeonAvailable())) {
10348 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10349 }
10350 return Register();
10351}
10352
10353Register fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10354 if (RetVT.SimpleTy != MVT::v8i16)
10355 return Register();
10356 if ((Subtarget->isNeonAvailable())) {
10357 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10358 }
10359 return Register();
10360}
10361
10362Register fastEmit_ISD_AND_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10363 if (RetVT.SimpleTy != MVT::v2i32)
10364 return Register();
10365 if ((Subtarget->isNeonAvailable())) {
10366 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10367 }
10368 return Register();
10369}
10370
10371Register fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10372 if (RetVT.SimpleTy != MVT::v4i32)
10373 return Register();
10374 if ((Subtarget->isNeonAvailable())) {
10375 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10376 }
10377 return Register();
10378}
10379
10380Register fastEmit_ISD_AND_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
10381 if (RetVT.SimpleTy != MVT::v1i64)
10382 return Register();
10383 if ((Subtarget->isNeonAvailable())) {
10384 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10385 }
10386 return Register();
10387}
10388
10389Register fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10390 if (RetVT.SimpleTy != MVT::v2i64)
10391 return Register();
10392 if ((Subtarget->isNeonAvailable())) {
10393 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10394 }
10395 return Register();
10396}
10397
10398Register fastEmit_ISD_AND_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10399 if (RetVT.SimpleTy != MVT::nxv16i8)
10400 return Register();
10401 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10402 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10403 }
10404 return Register();
10405}
10406
10407Register fastEmit_ISD_AND_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10408 if (RetVT.SimpleTy != MVT::nxv8i16)
10409 return Register();
10410 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10411 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10412 }
10413 return Register();
10414}
10415
10416Register fastEmit_ISD_AND_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10417 if (RetVT.SimpleTy != MVT::nxv4i32)
10418 return Register();
10419 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10420 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10421 }
10422 return Register();
10423}
10424
10425Register fastEmit_ISD_AND_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10426 if (RetVT.SimpleTy != MVT::nxv2i64)
10427 return Register();
10428 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10429 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10430 }
10431 return Register();
10432}
10433
10434Register fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10435 switch (VT.SimpleTy) {
10436 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
10437 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op1);
10438 case MVT::v8i8: return fastEmit_ISD_AND_MVT_v8i8_rr(RetVT, Op0, Op1);
10439 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
10440 case MVT::v4i16: return fastEmit_ISD_AND_MVT_v4i16_rr(RetVT, Op0, Op1);
10441 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
10442 case MVT::v2i32: return fastEmit_ISD_AND_MVT_v2i32_rr(RetVT, Op0, Op1);
10443 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
10444 case MVT::v1i64: return fastEmit_ISD_AND_MVT_v1i64_rr(RetVT, Op0, Op1);
10445 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
10446 case MVT::nxv16i8: return fastEmit_ISD_AND_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10447 case MVT::nxv8i16: return fastEmit_ISD_AND_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10448 case MVT::nxv4i32: return fastEmit_ISD_AND_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10449 case MVT::nxv2i64: return fastEmit_ISD_AND_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10450 default: return Register();
10451 }
10452}
10453
10454// FastEmit functions for ISD::AVGCEILS.
10455
10456Register fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10457 if (RetVT.SimpleTy != MVT::v8i8)
10458 return Register();
10459 if ((Subtarget->isNeonAvailable())) {
10460 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10461 }
10462 return Register();
10463}
10464
10465Register fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10466 if (RetVT.SimpleTy != MVT::v16i8)
10467 return Register();
10468 if ((Subtarget->isNeonAvailable())) {
10469 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10470 }
10471 return Register();
10472}
10473
10474Register fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10475 if (RetVT.SimpleTy != MVT::v4i16)
10476 return Register();
10477 if ((Subtarget->isNeonAvailable())) {
10478 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10479 }
10480 return Register();
10481}
10482
10483Register fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10484 if (RetVT.SimpleTy != MVT::v8i16)
10485 return Register();
10486 if ((Subtarget->isNeonAvailable())) {
10487 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10488 }
10489 return Register();
10490}
10491
10492Register fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10493 if (RetVT.SimpleTy != MVT::v2i32)
10494 return Register();
10495 if ((Subtarget->isNeonAvailable())) {
10496 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10497 }
10498 return Register();
10499}
10500
10501Register fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10502 if (RetVT.SimpleTy != MVT::v4i32)
10503 return Register();
10504 if ((Subtarget->isNeonAvailable())) {
10505 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10506 }
10507 return Register();
10508}
10509
10510Register fastEmit_ISD_AVGCEILS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10511 switch (VT.SimpleTy) {
10512 case MVT::v8i8: return fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(RetVT, Op0, Op1);
10513 case MVT::v16i8: return fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(RetVT, Op0, Op1);
10514 case MVT::v4i16: return fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(RetVT, Op0, Op1);
10515 case MVT::v8i16: return fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(RetVT, Op0, Op1);
10516 case MVT::v2i32: return fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(RetVT, Op0, Op1);
10517 case MVT::v4i32: return fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(RetVT, Op0, Op1);
10518 default: return Register();
10519 }
10520}
10521
10522// FastEmit functions for ISD::AVGCEILU.
10523
10524Register fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10525 if (RetVT.SimpleTy != MVT::v8i8)
10526 return Register();
10527 if ((Subtarget->isNeonAvailable())) {
10528 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10529 }
10530 return Register();
10531}
10532
10533Register fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10534 if (RetVT.SimpleTy != MVT::v16i8)
10535 return Register();
10536 if ((Subtarget->isNeonAvailable())) {
10537 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10538 }
10539 return Register();
10540}
10541
10542Register fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10543 if (RetVT.SimpleTy != MVT::v4i16)
10544 return Register();
10545 if ((Subtarget->isNeonAvailable())) {
10546 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10547 }
10548 return Register();
10549}
10550
10551Register fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10552 if (RetVT.SimpleTy != MVT::v8i16)
10553 return Register();
10554 if ((Subtarget->isNeonAvailable())) {
10555 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10556 }
10557 return Register();
10558}
10559
10560Register fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10561 if (RetVT.SimpleTy != MVT::v2i32)
10562 return Register();
10563 if ((Subtarget->isNeonAvailable())) {
10564 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10565 }
10566 return Register();
10567}
10568
10569Register fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10570 if (RetVT.SimpleTy != MVT::v4i32)
10571 return Register();
10572 if ((Subtarget->isNeonAvailable())) {
10573 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10574 }
10575 return Register();
10576}
10577
10578Register fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10579 switch (VT.SimpleTy) {
10580 case MVT::v8i8: return fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(RetVT, Op0, Op1);
10581 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
10582 case MVT::v4i16: return fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(RetVT, Op0, Op1);
10583 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
10584 case MVT::v2i32: return fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(RetVT, Op0, Op1);
10585 case MVT::v4i32: return fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(RetVT, Op0, Op1);
10586 default: return Register();
10587 }
10588}
10589
10590// FastEmit functions for ISD::AVGFLOORS.
10591
10592Register fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10593 if (RetVT.SimpleTy != MVT::v8i8)
10594 return Register();
10595 if ((Subtarget->isNeonAvailable())) {
10596 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10597 }
10598 return Register();
10599}
10600
10601Register fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10602 if (RetVT.SimpleTy != MVT::v16i8)
10603 return Register();
10604 if ((Subtarget->isNeonAvailable())) {
10605 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10606 }
10607 return Register();
10608}
10609
10610Register fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10611 if (RetVT.SimpleTy != MVT::v4i16)
10612 return Register();
10613 if ((Subtarget->isNeonAvailable())) {
10614 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10615 }
10616 return Register();
10617}
10618
10619Register fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10620 if (RetVT.SimpleTy != MVT::v8i16)
10621 return Register();
10622 if ((Subtarget->isNeonAvailable())) {
10623 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10624 }
10625 return Register();
10626}
10627
10628Register fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10629 if (RetVT.SimpleTy != MVT::v2i32)
10630 return Register();
10631 if ((Subtarget->isNeonAvailable())) {
10632 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10633 }
10634 return Register();
10635}
10636
10637Register fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10638 if (RetVT.SimpleTy != MVT::v4i32)
10639 return Register();
10640 if ((Subtarget->isNeonAvailable())) {
10641 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10642 }
10643 return Register();
10644}
10645
10646Register fastEmit_ISD_AVGFLOORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10647 switch (VT.SimpleTy) {
10648 case MVT::v8i8: return fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(RetVT, Op0, Op1);
10649 case MVT::v16i8: return fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(RetVT, Op0, Op1);
10650 case MVT::v4i16: return fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(RetVT, Op0, Op1);
10651 case MVT::v8i16: return fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(RetVT, Op0, Op1);
10652 case MVT::v2i32: return fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(RetVT, Op0, Op1);
10653 case MVT::v4i32: return fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(RetVT, Op0, Op1);
10654 default: return Register();
10655 }
10656}
10657
10658// FastEmit functions for ISD::AVGFLOORU.
10659
10660Register fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10661 if (RetVT.SimpleTy != MVT::v8i8)
10662 return Register();
10663 if ((Subtarget->isNeonAvailable())) {
10664 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10665 }
10666 return Register();
10667}
10668
10669Register fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10670 if (RetVT.SimpleTy != MVT::v16i8)
10671 return Register();
10672 if ((Subtarget->isNeonAvailable())) {
10673 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10674 }
10675 return Register();
10676}
10677
10678Register fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10679 if (RetVT.SimpleTy != MVT::v4i16)
10680 return Register();
10681 if ((Subtarget->isNeonAvailable())) {
10682 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10683 }
10684 return Register();
10685}
10686
10687Register fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10688 if (RetVT.SimpleTy != MVT::v8i16)
10689 return Register();
10690 if ((Subtarget->isNeonAvailable())) {
10691 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10692 }
10693 return Register();
10694}
10695
10696Register fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10697 if (RetVT.SimpleTy != MVT::v2i32)
10698 return Register();
10699 if ((Subtarget->isNeonAvailable())) {
10700 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10701 }
10702 return Register();
10703}
10704
10705Register fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10706 if (RetVT.SimpleTy != MVT::v4i32)
10707 return Register();
10708 if ((Subtarget->isNeonAvailable())) {
10709 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10710 }
10711 return Register();
10712}
10713
10714Register fastEmit_ISD_AVGFLOORU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10715 switch (VT.SimpleTy) {
10716 case MVT::v8i8: return fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(RetVT, Op0, Op1);
10717 case MVT::v16i8: return fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(RetVT, Op0, Op1);
10718 case MVT::v4i16: return fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(RetVT, Op0, Op1);
10719 case MVT::v8i16: return fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(RetVT, Op0, Op1);
10720 case MVT::v2i32: return fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(RetVT, Op0, Op1);
10721 case MVT::v4i32: return fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(RetVT, Op0, Op1);
10722 default: return Register();
10723 }
10724}
10725
10726// FastEmit functions for ISD::CLMUL.
10727
10728Register fastEmit_ISD_CLMUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10729 if (RetVT.SimpleTy != MVT::v8i8)
10730 return Register();
10731 if ((Subtarget->isNeonAvailable())) {
10732 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10733 }
10734 return Register();
10735}
10736
10737Register fastEmit_ISD_CLMUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10738 if (RetVT.SimpleTy != MVT::v16i8)
10739 return Register();
10740 if ((Subtarget->isNeonAvailable())) {
10741 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10742 }
10743 return Register();
10744}
10745
10746Register fastEmit_ISD_CLMUL_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10747 if (RetVT.SimpleTy != MVT::nxv16i8)
10748 return Register();
10749 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2() || Subtarget->hasSME()))) {
10750 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMUL_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
10751 }
10752 return Register();
10753}
10754
10755Register fastEmit_ISD_CLMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10756 switch (VT.SimpleTy) {
10757 case MVT::v8i8: return fastEmit_ISD_CLMUL_MVT_v8i8_rr(RetVT, Op0, Op1);
10758 case MVT::v16i8: return fastEmit_ISD_CLMUL_MVT_v16i8_rr(RetVT, Op0, Op1);
10759 case MVT::nxv16i8: return fastEmit_ISD_CLMUL_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10760 default: return Register();
10761 }
10762}
10763
10764// FastEmit functions for ISD::CONCAT_VECTORS.
10765
10766Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(MVT RetVT, Register Op0, Register Op1) {
10767 if (RetVT.SimpleTy != MVT::nxv2i1)
10768 return Register();
10769 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10770 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
10771 }
10772 return Register();
10773}
10774
10775Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
10776 if (RetVT.SimpleTy != MVT::nxv4i1)
10777 return Register();
10778 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10779 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
10780 }
10781 return Register();
10782}
10783
10784Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
10785 if (RetVT.SimpleTy != MVT::nxv8i1)
10786 return Register();
10787 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10788 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
10789 }
10790 return Register();
10791}
10792
10793Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
10794 if (RetVT.SimpleTy != MVT::nxv16i1)
10795 return Register();
10796 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10797 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
10798 }
10799 return Register();
10800}
10801
10802Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
10803 if (RetVT.SimpleTy != MVT::nxv4f16)
10804 return Register();
10805 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10806 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10807 }
10808 return Register();
10809}
10810
10811Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10812 if (RetVT.SimpleTy != MVT::nxv8f16)
10813 return Register();
10814 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10815 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10816 }
10817 return Register();
10818}
10819
10820Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10821 if (RetVT.SimpleTy != MVT::nxv4bf16)
10822 return Register();
10823 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10824 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10825 }
10826 return Register();
10827}
10828
10829Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10830 if (RetVT.SimpleTy != MVT::nxv8bf16)
10831 return Register();
10832 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10833 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10834 }
10835 return Register();
10836}
10837
10838Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10839 if (RetVT.SimpleTy != MVT::nxv4f32)
10840 return Register();
10841 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10842 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10843 }
10844 return Register();
10845}
10846
10847Register fastEmit_ISD_CONCAT_VECTORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10848 switch (VT.SimpleTy) {
10849 case MVT::nxv1i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(RetVT, Op0, Op1);
10850 case MVT::nxv2i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(RetVT, Op0, Op1);
10851 case MVT::nxv4i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(RetVT, Op0, Op1);
10852 case MVT::nxv8i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(RetVT, Op0, Op1);
10853 case MVT::nxv2f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(RetVT, Op0, Op1);
10854 case MVT::nxv4f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(RetVT, Op0, Op1);
10855 case MVT::nxv2bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
10856 case MVT::nxv4bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
10857 case MVT::nxv2f32: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(RetVT, Op0, Op1);
10858 default: return Register();
10859 }
10860}
10861
10862// FastEmit functions for ISD::FADD.
10863
10864Register fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10865 if (RetVT.SimpleTy != MVT::f16)
10866 return Register();
10867 if ((Subtarget->hasFullFP16())) {
10868 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
10869 }
10870 return Register();
10871}
10872
10873Register fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10874 if (RetVT.SimpleTy != MVT::f32)
10875 return Register();
10876 if ((Subtarget->hasFPARMv8())) {
10877 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
10878 }
10879 return Register();
10880}
10881
10882Register fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
10883 if (RetVT.SimpleTy != MVT::f64)
10884 return Register();
10885 if ((Subtarget->hasFPARMv8())) {
10886 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10887 }
10888 return Register();
10889}
10890
10891Register fastEmit_ISD_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10892 if (RetVT.SimpleTy != MVT::v4f16)
10893 return Register();
10894 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10895 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10896 }
10897 return Register();
10898}
10899
10900Register fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10901 if (RetVT.SimpleTy != MVT::v8f16)
10902 return Register();
10903 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10904 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10905 }
10906 return Register();
10907}
10908
10909Register fastEmit_ISD_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10910 if (RetVT.SimpleTy != MVT::v2f32)
10911 return Register();
10912 if ((Subtarget->isNeonAvailable())) {
10913 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10914 }
10915 return Register();
10916}
10917
10918Register fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10919 if (RetVT.SimpleTy != MVT::v4f32)
10920 return Register();
10921 if ((Subtarget->isNeonAvailable())) {
10922 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10923 }
10924 return Register();
10925}
10926
10927Register fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10928 if (RetVT.SimpleTy != MVT::v2f64)
10929 return Register();
10930 if ((Subtarget->isNeonAvailable())) {
10931 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10932 }
10933 return Register();
10934}
10935
10936Register fastEmit_ISD_FADD_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10937 if (RetVT.SimpleTy != MVT::nxv8f16)
10938 return Register();
10939 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10940 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10941 }
10942 return Register();
10943}
10944
10945Register fastEmit_ISD_FADD_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10946 if (RetVT.SimpleTy != MVT::nxv8bf16)
10947 return Register();
10948 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
10949 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFADD_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10950 }
10951 return Register();
10952}
10953
10954Register fastEmit_ISD_FADD_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10955 if (RetVT.SimpleTy != MVT::nxv4f32)
10956 return Register();
10957 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10958 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10959 }
10960 return Register();
10961}
10962
10963Register fastEmit_ISD_FADD_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10964 if (RetVT.SimpleTy != MVT::nxv2f64)
10965 return Register();
10966 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10967 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10968 }
10969 return Register();
10970}
10971
10972Register fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10973 switch (VT.SimpleTy) {
10974 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
10975 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
10976 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
10977 case MVT::v4f16: return fastEmit_ISD_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
10978 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
10979 case MVT::v2f32: return fastEmit_ISD_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
10980 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
10981 case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
10982 case MVT::nxv8f16: return fastEmit_ISD_FADD_MVT_nxv8f16_rr(RetVT, Op0, Op1);
10983 case MVT::nxv8bf16: return fastEmit_ISD_FADD_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
10984 case MVT::nxv4f32: return fastEmit_ISD_FADD_MVT_nxv4f32_rr(RetVT, Op0, Op1);
10985 case MVT::nxv2f64: return fastEmit_ISD_FADD_MVT_nxv2f64_rr(RetVT, Op0, Op1);
10986 default: return Register();
10987 }
10988}
10989
10990// FastEmit functions for ISD::FDIV.
10991
10992Register fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10993 if (RetVT.SimpleTy != MVT::f16)
10994 return Register();
10995 if ((Subtarget->hasFullFP16())) {
10996 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
10997 }
10998 return Register();
10999}
11000
11001Register fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11002 if (RetVT.SimpleTy != MVT::f32)
11003 return Register();
11004 if ((Subtarget->hasFPARMv8())) {
11005 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11006 }
11007 return Register();
11008}
11009
11010Register fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11011 if (RetVT.SimpleTy != MVT::f64)
11012 return Register();
11013 if ((Subtarget->hasFPARMv8())) {
11014 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11015 }
11016 return Register();
11017}
11018
11019Register fastEmit_ISD_FDIV_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11020 if (RetVT.SimpleTy != MVT::v4f16)
11021 return Register();
11022 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11023 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11024 }
11025 return Register();
11026}
11027
11028Register fastEmit_ISD_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11029 if (RetVT.SimpleTy != MVT::v8f16)
11030 return Register();
11031 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11032 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11033 }
11034 return Register();
11035}
11036
11037Register fastEmit_ISD_FDIV_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11038 if (RetVT.SimpleTy != MVT::v2f32)
11039 return Register();
11040 if ((Subtarget->isNeonAvailable())) {
11041 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11042 }
11043 return Register();
11044}
11045
11046Register fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11047 if (RetVT.SimpleTy != MVT::v4f32)
11048 return Register();
11049 if ((Subtarget->isNeonAvailable())) {
11050 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11051 }
11052 return Register();
11053}
11054
11055Register fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11056 if (RetVT.SimpleTy != MVT::v2f64)
11057 return Register();
11058 if ((Subtarget->isNeonAvailable())) {
11059 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11060 }
11061 return Register();
11062}
11063
11064Register fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11065 switch (VT.SimpleTy) {
11066 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
11067 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
11068 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
11069 case MVT::v4f16: return fastEmit_ISD_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1);
11070 case MVT::v8f16: return fastEmit_ISD_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
11071 case MVT::v2f32: return fastEmit_ISD_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1);
11072 case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
11073 case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
11074 default: return Register();
11075 }
11076}
11077
11078// FastEmit functions for ISD::FMAXIMUM.
11079
11080Register fastEmit_ISD_FMAXIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11081 if (RetVT.SimpleTy != MVT::f16)
11082 return Register();
11083 if ((Subtarget->hasFullFP16())) {
11084 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11085 }
11086 return Register();
11087}
11088
11089Register fastEmit_ISD_FMAXIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11090 if (RetVT.SimpleTy != MVT::f32)
11091 return Register();
11092 if ((Subtarget->hasFPARMv8())) {
11093 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11094 }
11095 return Register();
11096}
11097
11098Register fastEmit_ISD_FMAXIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11099 if (RetVT.SimpleTy != MVT::f64)
11100 return Register();
11101 if ((Subtarget->hasFPARMv8())) {
11102 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11103 }
11104 return Register();
11105}
11106
11107Register fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11108 if (RetVT.SimpleTy != MVT::v4f16)
11109 return Register();
11110 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11111 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11112 }
11113 return Register();
11114}
11115
11116Register fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11117 if (RetVT.SimpleTy != MVT::v8f16)
11118 return Register();
11119 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11120 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11121 }
11122 return Register();
11123}
11124
11125Register fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11126 if (RetVT.SimpleTy != MVT::v2f32)
11127 return Register();
11128 if ((Subtarget->isNeonAvailable())) {
11129 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11130 }
11131 return Register();
11132}
11133
11134Register fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11135 if (RetVT.SimpleTy != MVT::v4f32)
11136 return Register();
11137 if ((Subtarget->isNeonAvailable())) {
11138 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11139 }
11140 return Register();
11141}
11142
11143Register fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11144 if (RetVT.SimpleTy != MVT::v1f64)
11145 return Register();
11146 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11147}
11148
11149Register fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11150 if (RetVT.SimpleTy != MVT::v2f64)
11151 return Register();
11152 if ((Subtarget->isNeonAvailable())) {
11153 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11154 }
11155 return Register();
11156}
11157
11158Register fastEmit_ISD_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11159 switch (VT.SimpleTy) {
11160 case MVT::f16: return fastEmit_ISD_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1);
11161 case MVT::f32: return fastEmit_ISD_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1);
11162 case MVT::f64: return fastEmit_ISD_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1);
11163 case MVT::v4f16: return fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11164 case MVT::v8f16: return fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11165 case MVT::v2f32: return fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11166 case MVT::v4f32: return fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11167 case MVT::v1f64: return fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11168 case MVT::v2f64: return fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11169 default: return Register();
11170 }
11171}
11172
11173// FastEmit functions for ISD::FMAXNUM.
11174
11175Register fastEmit_ISD_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11176 if (RetVT.SimpleTy != MVT::f16)
11177 return Register();
11178 if ((Subtarget->hasFullFP16())) {
11179 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11180 }
11181 return Register();
11182}
11183
11184Register fastEmit_ISD_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11185 if (RetVT.SimpleTy != MVT::f32)
11186 return Register();
11187 if ((Subtarget->hasFPARMv8())) {
11188 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11189 }
11190 return Register();
11191}
11192
11193Register fastEmit_ISD_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11194 if (RetVT.SimpleTy != MVT::f64)
11195 return Register();
11196 if ((Subtarget->hasFPARMv8())) {
11197 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11198 }
11199 return Register();
11200}
11201
11202Register fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11203 if (RetVT.SimpleTy != MVT::v4f16)
11204 return Register();
11205 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11206 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11207 }
11208 return Register();
11209}
11210
11211Register fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11212 if (RetVT.SimpleTy != MVT::v8f16)
11213 return Register();
11214 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11215 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11216 }
11217 return Register();
11218}
11219
11220Register fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11221 if (RetVT.SimpleTy != MVT::v2f32)
11222 return Register();
11223 if ((Subtarget->isNeonAvailable())) {
11224 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11225 }
11226 return Register();
11227}
11228
11229Register fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11230 if (RetVT.SimpleTy != MVT::v4f32)
11231 return Register();
11232 if ((Subtarget->isNeonAvailable())) {
11233 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11234 }
11235 return Register();
11236}
11237
11238Register fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11239 if (RetVT.SimpleTy != MVT::v1f64)
11240 return Register();
11241 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11242}
11243
11244Register fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11245 if (RetVT.SimpleTy != MVT::v2f64)
11246 return Register();
11247 if ((Subtarget->isNeonAvailable())) {
11248 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11249 }
11250 return Register();
11251}
11252
11253Register fastEmit_ISD_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11254 switch (VT.SimpleTy) {
11255 case MVT::f16: return fastEmit_ISD_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
11256 case MVT::f32: return fastEmit_ISD_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
11257 case MVT::f64: return fastEmit_ISD_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
11258 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11259 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11260 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11261 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11262 case MVT::v1f64: return fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11263 case MVT::v2f64: return fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11264 default: return Register();
11265 }
11266}
11267
11268// FastEmit functions for ISD::FMAXNUM_IEEE.
11269
11270Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11271 if (RetVT.SimpleTy != MVT::f16)
11272 return Register();
11273 if ((Subtarget->hasFullFP16())) {
11274 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11275 }
11276 return Register();
11277}
11278
11279Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11280 if (RetVT.SimpleTy != MVT::f32)
11281 return Register();
11282 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11283}
11284
11285Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11286 if (RetVT.SimpleTy != MVT::f64)
11287 return Register();
11288 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11289}
11290
11291Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11292 if (RetVT.SimpleTy != MVT::v4f16)
11293 return Register();
11294 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11295 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11296 }
11297 return Register();
11298}
11299
11300Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11301 if (RetVT.SimpleTy != MVT::v8f16)
11302 return Register();
11303 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11304 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11305 }
11306 return Register();
11307}
11308
11309Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11310 if (RetVT.SimpleTy != MVT::v2f32)
11311 return Register();
11312 if ((Subtarget->isNeonAvailable())) {
11313 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11314 }
11315 return Register();
11316}
11317
11318Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11319 if (RetVT.SimpleTy != MVT::v4f32)
11320 return Register();
11321 if ((Subtarget->isNeonAvailable())) {
11322 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11323 }
11324 return Register();
11325}
11326
11327Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11328 if (RetVT.SimpleTy != MVT::v2f64)
11329 return Register();
11330 if ((Subtarget->isNeonAvailable())) {
11331 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11332 }
11333 return Register();
11334}
11335
11336Register fastEmit_ISD_FMAXNUM_IEEE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11337 switch (VT.SimpleTy) {
11338 case MVT::f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f16_rr(RetVT, Op0, Op1);
11339 case MVT::f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f32_rr(RetVT, Op0, Op1);
11340 case MVT::f64: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f64_rr(RetVT, Op0, Op1);
11341 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f16_rr(RetVT, Op0, Op1);
11342 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v8f16_rr(RetVT, Op0, Op1);
11343 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f32_rr(RetVT, Op0, Op1);
11344 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f32_rr(RetVT, Op0, Op1);
11345 case MVT::v2f64: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f64_rr(RetVT, Op0, Op1);
11346 default: return Register();
11347 }
11348}
11349
11350// FastEmit functions for ISD::FMINIMUM.
11351
11352Register fastEmit_ISD_FMINIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11353 if (RetVT.SimpleTy != MVT::f16)
11354 return Register();
11355 if ((Subtarget->hasFullFP16())) {
11356 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11357 }
11358 return Register();
11359}
11360
11361Register fastEmit_ISD_FMINIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11362 if (RetVT.SimpleTy != MVT::f32)
11363 return Register();
11364 if ((Subtarget->hasFPARMv8())) {
11365 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11366 }
11367 return Register();
11368}
11369
11370Register fastEmit_ISD_FMINIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11371 if (RetVT.SimpleTy != MVT::f64)
11372 return Register();
11373 if ((Subtarget->hasFPARMv8())) {
11374 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11375 }
11376 return Register();
11377}
11378
11379Register fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11380 if (RetVT.SimpleTy != MVT::v4f16)
11381 return Register();
11382 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11383 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11384 }
11385 return Register();
11386}
11387
11388Register fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11389 if (RetVT.SimpleTy != MVT::v8f16)
11390 return Register();
11391 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11392 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11393 }
11394 return Register();
11395}
11396
11397Register fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11398 if (RetVT.SimpleTy != MVT::v2f32)
11399 return Register();
11400 if ((Subtarget->isNeonAvailable())) {
11401 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11402 }
11403 return Register();
11404}
11405
11406Register fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11407 if (RetVT.SimpleTy != MVT::v4f32)
11408 return Register();
11409 if ((Subtarget->isNeonAvailable())) {
11410 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11411 }
11412 return Register();
11413}
11414
11415Register fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11416 if (RetVT.SimpleTy != MVT::v1f64)
11417 return Register();
11418 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11419}
11420
11421Register fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11422 if (RetVT.SimpleTy != MVT::v2f64)
11423 return Register();
11424 if ((Subtarget->isNeonAvailable())) {
11425 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11426 }
11427 return Register();
11428}
11429
11430Register fastEmit_ISD_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11431 switch (VT.SimpleTy) {
11432 case MVT::f16: return fastEmit_ISD_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1);
11433 case MVT::f32: return fastEmit_ISD_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1);
11434 case MVT::f64: return fastEmit_ISD_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1);
11435 case MVT::v4f16: return fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11436 case MVT::v8f16: return fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11437 case MVT::v2f32: return fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11438 case MVT::v4f32: return fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11439 case MVT::v1f64: return fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11440 case MVT::v2f64: return fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11441 default: return Register();
11442 }
11443}
11444
11445// FastEmit functions for ISD::FMINNUM.
11446
11447Register fastEmit_ISD_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11448 if (RetVT.SimpleTy != MVT::f16)
11449 return Register();
11450 if ((Subtarget->hasFullFP16())) {
11451 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11452 }
11453 return Register();
11454}
11455
11456Register fastEmit_ISD_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11457 if (RetVT.SimpleTy != MVT::f32)
11458 return Register();
11459 if ((Subtarget->hasFPARMv8())) {
11460 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11461 }
11462 return Register();
11463}
11464
11465Register fastEmit_ISD_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11466 if (RetVT.SimpleTy != MVT::f64)
11467 return Register();
11468 if ((Subtarget->hasFPARMv8())) {
11469 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11470 }
11471 return Register();
11472}
11473
11474Register fastEmit_ISD_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11475 if (RetVT.SimpleTy != MVT::v4f16)
11476 return Register();
11477 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11478 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11479 }
11480 return Register();
11481}
11482
11483Register fastEmit_ISD_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11484 if (RetVT.SimpleTy != MVT::v8f16)
11485 return Register();
11486 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11487 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11488 }
11489 return Register();
11490}
11491
11492Register fastEmit_ISD_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11493 if (RetVT.SimpleTy != MVT::v2f32)
11494 return Register();
11495 if ((Subtarget->isNeonAvailable())) {
11496 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11497 }
11498 return Register();
11499}
11500
11501Register fastEmit_ISD_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11502 if (RetVT.SimpleTy != MVT::v4f32)
11503 return Register();
11504 if ((Subtarget->isNeonAvailable())) {
11505 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11506 }
11507 return Register();
11508}
11509
11510Register fastEmit_ISD_FMINNUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11511 if (RetVT.SimpleTy != MVT::v1f64)
11512 return Register();
11513 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11514}
11515
11516Register fastEmit_ISD_FMINNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11517 if (RetVT.SimpleTy != MVT::v2f64)
11518 return Register();
11519 if ((Subtarget->isNeonAvailable())) {
11520 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11521 }
11522 return Register();
11523}
11524
11525Register fastEmit_ISD_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11526 switch (VT.SimpleTy) {
11527 case MVT::f16: return fastEmit_ISD_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
11528 case MVT::f32: return fastEmit_ISD_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
11529 case MVT::f64: return fastEmit_ISD_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
11530 case MVT::v4f16: return fastEmit_ISD_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11531 case MVT::v8f16: return fastEmit_ISD_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11532 case MVT::v2f32: return fastEmit_ISD_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11533 case MVT::v4f32: return fastEmit_ISD_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11534 case MVT::v1f64: return fastEmit_ISD_FMINNUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11535 case MVT::v2f64: return fastEmit_ISD_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11536 default: return Register();
11537 }
11538}
11539
11540// FastEmit functions for ISD::FMINNUM_IEEE.
11541
11542Register fastEmit_ISD_FMINNUM_IEEE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11543 if (RetVT.SimpleTy != MVT::f16)
11544 return Register();
11545 if ((Subtarget->hasFullFP16())) {
11546 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11547 }
11548 return Register();
11549}
11550
11551Register fastEmit_ISD_FMINNUM_IEEE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11552 if (RetVT.SimpleTy != MVT::f32)
11553 return Register();
11554 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11555}
11556
11557Register fastEmit_ISD_FMINNUM_IEEE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11558 if (RetVT.SimpleTy != MVT::f64)
11559 return Register();
11560 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11561}
11562
11563Register fastEmit_ISD_FMINNUM_IEEE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11564 if (RetVT.SimpleTy != MVT::v4f16)
11565 return Register();
11566 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11567 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11568 }
11569 return Register();
11570}
11571
11572Register fastEmit_ISD_FMINNUM_IEEE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11573 if (RetVT.SimpleTy != MVT::v8f16)
11574 return Register();
11575 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11576 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11577 }
11578 return Register();
11579}
11580
11581Register fastEmit_ISD_FMINNUM_IEEE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11582 if (RetVT.SimpleTy != MVT::v2f32)
11583 return Register();
11584 if ((Subtarget->isNeonAvailable())) {
11585 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11586 }
11587 return Register();
11588}
11589
11590Register fastEmit_ISD_FMINNUM_IEEE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11591 if (RetVT.SimpleTy != MVT::v4f32)
11592 return Register();
11593 if ((Subtarget->isNeonAvailable())) {
11594 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11595 }
11596 return Register();
11597}
11598
11599Register fastEmit_ISD_FMINNUM_IEEE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11600 if (RetVT.SimpleTy != MVT::v2f64)
11601 return Register();
11602 if ((Subtarget->isNeonAvailable())) {
11603 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11604 }
11605 return Register();
11606}
11607
11608Register fastEmit_ISD_FMINNUM_IEEE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11609 switch (VT.SimpleTy) {
11610 case MVT::f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_f16_rr(RetVT, Op0, Op1);
11611 case MVT::f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_f32_rr(RetVT, Op0, Op1);
11612 case MVT::f64: return fastEmit_ISD_FMINNUM_IEEE_MVT_f64_rr(RetVT, Op0, Op1);
11613 case MVT::v4f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_v4f16_rr(RetVT, Op0, Op1);
11614 case MVT::v8f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_v8f16_rr(RetVT, Op0, Op1);
11615 case MVT::v2f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_v2f32_rr(RetVT, Op0, Op1);
11616 case MVT::v4f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_v4f32_rr(RetVT, Op0, Op1);
11617 case MVT::v2f64: return fastEmit_ISD_FMINNUM_IEEE_MVT_v2f64_rr(RetVT, Op0, Op1);
11618 default: return Register();
11619 }
11620}
11621
11622// FastEmit functions for ISD::FMUL.
11623
11624Register fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11625 if (RetVT.SimpleTy != MVT::f16)
11626 return Register();
11627 if ((Subtarget->hasFullFP16())) {
11628 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11629 }
11630 return Register();
11631}
11632
11633Register fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11634 if (RetVT.SimpleTy != MVT::f32)
11635 return Register();
11636 if ((Subtarget->hasFPARMv8())) {
11637 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11638 }
11639 return Register();
11640}
11641
11642Register fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11643 if (RetVT.SimpleTy != MVT::f64)
11644 return Register();
11645 if ((Subtarget->hasFPARMv8())) {
11646 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11647 }
11648 return Register();
11649}
11650
11651Register fastEmit_ISD_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11652 if (RetVT.SimpleTy != MVT::v4f16)
11653 return Register();
11654 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11655 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11656 }
11657 return Register();
11658}
11659
11660Register fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11661 if (RetVT.SimpleTy != MVT::v8f16)
11662 return Register();
11663 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11664 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11665 }
11666 return Register();
11667}
11668
11669Register fastEmit_ISD_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11670 if (RetVT.SimpleTy != MVT::v2f32)
11671 return Register();
11672 if ((Subtarget->isNeonAvailable())) {
11673 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11674 }
11675 return Register();
11676}
11677
11678Register fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11679 if (RetVT.SimpleTy != MVT::v4f32)
11680 return Register();
11681 if ((Subtarget->isNeonAvailable())) {
11682 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11683 }
11684 return Register();
11685}
11686
11687Register fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11688 if (RetVT.SimpleTy != MVT::v2f64)
11689 return Register();
11690 if ((Subtarget->isNeonAvailable())) {
11691 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11692 }
11693 return Register();
11694}
11695
11696Register fastEmit_ISD_FMUL_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11697 if (RetVT.SimpleTy != MVT::nxv8f16)
11698 return Register();
11699 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11700 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11701 }
11702 return Register();
11703}
11704
11705Register fastEmit_ISD_FMUL_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11706 if (RetVT.SimpleTy != MVT::nxv8bf16)
11707 return Register();
11708 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
11709 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFMUL_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11710 }
11711 return Register();
11712}
11713
11714Register fastEmit_ISD_FMUL_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11715 if (RetVT.SimpleTy != MVT::nxv4f32)
11716 return Register();
11717 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11718 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11719 }
11720 return Register();
11721}
11722
11723Register fastEmit_ISD_FMUL_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11724 if (RetVT.SimpleTy != MVT::nxv2f64)
11725 return Register();
11726 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11727 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11728 }
11729 return Register();
11730}
11731
11732Register fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11733 switch (VT.SimpleTy) {
11734 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
11735 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
11736 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
11737 case MVT::v4f16: return fastEmit_ISD_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
11738 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
11739 case MVT::v2f32: return fastEmit_ISD_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
11740 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
11741 case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
11742 case MVT::nxv8f16: return fastEmit_ISD_FMUL_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11743 case MVT::nxv8bf16: return fastEmit_ISD_FMUL_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11744 case MVT::nxv4f32: return fastEmit_ISD_FMUL_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11745 case MVT::nxv2f64: return fastEmit_ISD_FMUL_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11746 default: return Register();
11747 }
11748}
11749
11750// FastEmit functions for ISD::FSUB.
11751
11752Register fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11753 if (RetVT.SimpleTy != MVT::f16)
11754 return Register();
11755 if ((Subtarget->hasFullFP16())) {
11756 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11757 }
11758 return Register();
11759}
11760
11761Register fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11762 if (RetVT.SimpleTy != MVT::f32)
11763 return Register();
11764 if ((Subtarget->hasFPARMv8())) {
11765 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11766 }
11767 return Register();
11768}
11769
11770Register fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11771 if (RetVT.SimpleTy != MVT::f64)
11772 return Register();
11773 if ((Subtarget->hasFPARMv8())) {
11774 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11775 }
11776 return Register();
11777}
11778
11779Register fastEmit_ISD_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11780 if (RetVT.SimpleTy != MVT::v4f16)
11781 return Register();
11782 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11783 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11784 }
11785 return Register();
11786}
11787
11788Register fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11789 if (RetVT.SimpleTy != MVT::v8f16)
11790 return Register();
11791 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11792 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11793 }
11794 return Register();
11795}
11796
11797Register fastEmit_ISD_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11798 if (RetVT.SimpleTy != MVT::v2f32)
11799 return Register();
11800 if ((Subtarget->isNeonAvailable())) {
11801 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11802 }
11803 return Register();
11804}
11805
11806Register fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11807 if (RetVT.SimpleTy != MVT::v4f32)
11808 return Register();
11809 if ((Subtarget->isNeonAvailable())) {
11810 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11811 }
11812 return Register();
11813}
11814
11815Register fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11816 if (RetVT.SimpleTy != MVT::v2f64)
11817 return Register();
11818 if ((Subtarget->isNeonAvailable())) {
11819 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11820 }
11821 return Register();
11822}
11823
11824Register fastEmit_ISD_FSUB_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11825 if (RetVT.SimpleTy != MVT::nxv8f16)
11826 return Register();
11827 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11828 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11829 }
11830 return Register();
11831}
11832
11833Register fastEmit_ISD_FSUB_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11834 if (RetVT.SimpleTy != MVT::nxv8bf16)
11835 return Register();
11836 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
11837 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFSUB_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11838 }
11839 return Register();
11840}
11841
11842Register fastEmit_ISD_FSUB_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11843 if (RetVT.SimpleTy != MVT::nxv4f32)
11844 return Register();
11845 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11846 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11847 }
11848 return Register();
11849}
11850
11851Register fastEmit_ISD_FSUB_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11852 if (RetVT.SimpleTy != MVT::nxv2f64)
11853 return Register();
11854 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11855 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11856 }
11857 return Register();
11858}
11859
11860Register fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11861 switch (VT.SimpleTy) {
11862 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
11863 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
11864 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
11865 case MVT::v4f16: return fastEmit_ISD_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
11866 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
11867 case MVT::v2f32: return fastEmit_ISD_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
11868 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
11869 case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
11870 case MVT::nxv8f16: return fastEmit_ISD_FSUB_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11871 case MVT::nxv8bf16: return fastEmit_ISD_FSUB_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11872 case MVT::nxv4f32: return fastEmit_ISD_FSUB_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11873 case MVT::nxv2f64: return fastEmit_ISD_FSUB_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11874 default: return Register();
11875 }
11876}
11877
11878// FastEmit functions for ISD::GET_ACTIVE_LANE_MASK.
11879
11880Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv2i1_rr(Register Op0, Register Op1) {
11881 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11882 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_D, RC: &AArch64::PPRRegClass, Op0, Op1);
11883 }
11884 return Register();
11885}
11886
11887Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv4i1_rr(Register Op0, Register Op1) {
11888 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11889 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_S, RC: &AArch64::PPRRegClass, Op0, Op1);
11890 }
11891 return Register();
11892}
11893
11894Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv8i1_rr(Register Op0, Register Op1) {
11895 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11896 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_H, RC: &AArch64::PPRRegClass, Op0, Op1);
11897 }
11898 return Register();
11899}
11900
11901Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv16i1_rr(Register Op0, Register Op1) {
11902 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11903 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_B, RC: &AArch64::PPRRegClass, Op0, Op1);
11904 }
11905 return Register();
11906}
11907
11908Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
11909switch (RetVT.SimpleTy) {
11910 case MVT::nxv2i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv2i1_rr(Op0, Op1);
11911 case MVT::nxv4i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv4i1_rr(Op0, Op1);
11912 case MVT::nxv8i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv8i1_rr(Op0, Op1);
11913 case MVT::nxv16i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv16i1_rr(Op0, Op1);
11914 default: return Register();
11915}
11916}
11917
11918Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv2i1_rr(Register Op0, Register Op1) {
11919 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11920 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_D, RC: &AArch64::PPRRegClass, Op0, Op1);
11921 }
11922 return Register();
11923}
11924
11925Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv4i1_rr(Register Op0, Register Op1) {
11926 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11927 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_S, RC: &AArch64::PPRRegClass, Op0, Op1);
11928 }
11929 return Register();
11930}
11931
11932Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv8i1_rr(Register Op0, Register Op1) {
11933 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11934 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_H, RC: &AArch64::PPRRegClass, Op0, Op1);
11935 }
11936 return Register();
11937}
11938
11939Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv16i1_rr(Register Op0, Register Op1) {
11940 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11941 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_B, RC: &AArch64::PPRRegClass, Op0, Op1);
11942 }
11943 return Register();
11944}
11945
11946Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11947switch (RetVT.SimpleTy) {
11948 case MVT::nxv2i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv2i1_rr(Op0, Op1);
11949 case MVT::nxv4i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv4i1_rr(Op0, Op1);
11950 case MVT::nxv8i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv8i1_rr(Op0, Op1);
11951 case MVT::nxv16i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv16i1_rr(Op0, Op1);
11952 default: return Register();
11953}
11954}
11955
11956Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11957 switch (VT.SimpleTy) {
11958 case MVT::i32: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_rr(RetVT, Op0, Op1);
11959 case MVT::i64: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_rr(RetVT, Op0, Op1);
11960 default: return Register();
11961 }
11962}
11963
11964// FastEmit functions for ISD::MUL.
11965
11966Register fastEmit_ISD_MUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
11967 if (RetVT.SimpleTy != MVT::v8i8)
11968 return Register();
11969 if ((Subtarget->isNeonAvailable())) {
11970 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
11971 }
11972 return Register();
11973}
11974
11975Register fastEmit_ISD_MUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
11976 if (RetVT.SimpleTy != MVT::v16i8)
11977 return Register();
11978 if ((Subtarget->isNeonAvailable())) {
11979 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
11980 }
11981 return Register();
11982}
11983
11984Register fastEmit_ISD_MUL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
11985 if (RetVT.SimpleTy != MVT::v4i16)
11986 return Register();
11987 if ((Subtarget->isNeonAvailable())) {
11988 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11989 }
11990 return Register();
11991}
11992
11993Register fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
11994 if (RetVT.SimpleTy != MVT::v8i16)
11995 return Register();
11996 if ((Subtarget->isNeonAvailable())) {
11997 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11998 }
11999 return Register();
12000}
12001
12002Register fastEmit_ISD_MUL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12003 if (RetVT.SimpleTy != MVT::v2i32)
12004 return Register();
12005 if ((Subtarget->isNeonAvailable())) {
12006 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12007 }
12008 return Register();
12009}
12010
12011Register fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12012 if (RetVT.SimpleTy != MVT::v4i32)
12013 return Register();
12014 if ((Subtarget->isNeonAvailable())) {
12015 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12016 }
12017 return Register();
12018}
12019
12020Register fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12021 switch (VT.SimpleTy) {
12022 case MVT::v8i8: return fastEmit_ISD_MUL_MVT_v8i8_rr(RetVT, Op0, Op1);
12023 case MVT::v16i8: return fastEmit_ISD_MUL_MVT_v16i8_rr(RetVT, Op0, Op1);
12024 case MVT::v4i16: return fastEmit_ISD_MUL_MVT_v4i16_rr(RetVT, Op0, Op1);
12025 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
12026 case MVT::v2i32: return fastEmit_ISD_MUL_MVT_v2i32_rr(RetVT, Op0, Op1);
12027 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
12028 default: return Register();
12029 }
12030}
12031
12032// FastEmit functions for ISD::MULHS.
12033
12034Register fastEmit_ISD_MULHS_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12035 if (RetVT.SimpleTy != MVT::i64)
12036 return Register();
12037 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULHrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12038}
12039
12040Register fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12041 switch (VT.SimpleTy) {
12042 case MVT::i64: return fastEmit_ISD_MULHS_MVT_i64_rr(RetVT, Op0, Op1);
12043 default: return Register();
12044 }
12045}
12046
12047// FastEmit functions for ISD::MULHU.
12048
12049Register fastEmit_ISD_MULHU_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12050 if (RetVT.SimpleTy != MVT::i64)
12051 return Register();
12052 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULHrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12053}
12054
12055Register fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12056 switch (VT.SimpleTy) {
12057 case MVT::i64: return fastEmit_ISD_MULHU_MVT_i64_rr(RetVT, Op0, Op1);
12058 default: return Register();
12059 }
12060}
12061
12062// FastEmit functions for ISD::OR.
12063
12064Register fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12065 if (RetVT.SimpleTy != MVT::i32)
12066 return Register();
12067 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12068}
12069
12070Register fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12071 if (RetVT.SimpleTy != MVT::i64)
12072 return Register();
12073 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12074}
12075
12076Register fastEmit_ISD_OR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12077 if (RetVT.SimpleTy != MVT::v8i8)
12078 return Register();
12079 if ((Subtarget->isNeonAvailable())) {
12080 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12081 }
12082 return Register();
12083}
12084
12085Register fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12086 if (RetVT.SimpleTy != MVT::v16i8)
12087 return Register();
12088 if ((Subtarget->isNeonAvailable())) {
12089 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12090 }
12091 return Register();
12092}
12093
12094Register fastEmit_ISD_OR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12095 if (RetVT.SimpleTy != MVT::v4i16)
12096 return Register();
12097 if ((Subtarget->isNeonAvailable())) {
12098 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12099 }
12100 return Register();
12101}
12102
12103Register fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12104 if (RetVT.SimpleTy != MVT::v8i16)
12105 return Register();
12106 if ((Subtarget->isNeonAvailable())) {
12107 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12108 }
12109 return Register();
12110}
12111
12112Register fastEmit_ISD_OR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12113 if (RetVT.SimpleTy != MVT::v2i32)
12114 return Register();
12115 if ((Subtarget->isNeonAvailable())) {
12116 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12117 }
12118 return Register();
12119}
12120
12121Register fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12122 if (RetVT.SimpleTy != MVT::v4i32)
12123 return Register();
12124 if ((Subtarget->isNeonAvailable())) {
12125 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12126 }
12127 return Register();
12128}
12129
12130Register fastEmit_ISD_OR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12131 if (RetVT.SimpleTy != MVT::v1i64)
12132 return Register();
12133 if ((Subtarget->isNeonAvailable())) {
12134 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12135 }
12136 return Register();
12137}
12138
12139Register fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12140 if (RetVT.SimpleTy != MVT::v2i64)
12141 return Register();
12142 if ((Subtarget->isNeonAvailable())) {
12143 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12144 }
12145 return Register();
12146}
12147
12148Register fastEmit_ISD_OR_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12149 if (RetVT.SimpleTy != MVT::nxv16i8)
12150 return Register();
12151 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12152 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12153 }
12154 return Register();
12155}
12156
12157Register fastEmit_ISD_OR_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12158 if (RetVT.SimpleTy != MVT::nxv8i16)
12159 return Register();
12160 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12161 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12162 }
12163 return Register();
12164}
12165
12166Register fastEmit_ISD_OR_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12167 if (RetVT.SimpleTy != MVT::nxv4i32)
12168 return Register();
12169 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12170 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12171 }
12172 return Register();
12173}
12174
12175Register fastEmit_ISD_OR_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12176 if (RetVT.SimpleTy != MVT::nxv2i64)
12177 return Register();
12178 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12179 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12180 }
12181 return Register();
12182}
12183
12184Register fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12185 switch (VT.SimpleTy) {
12186 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
12187 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op1);
12188 case MVT::v8i8: return fastEmit_ISD_OR_MVT_v8i8_rr(RetVT, Op0, Op1);
12189 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
12190 case MVT::v4i16: return fastEmit_ISD_OR_MVT_v4i16_rr(RetVT, Op0, Op1);
12191 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
12192 case MVT::v2i32: return fastEmit_ISD_OR_MVT_v2i32_rr(RetVT, Op0, Op1);
12193 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
12194 case MVT::v1i64: return fastEmit_ISD_OR_MVT_v1i64_rr(RetVT, Op0, Op1);
12195 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
12196 case MVT::nxv16i8: return fastEmit_ISD_OR_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12197 case MVT::nxv8i16: return fastEmit_ISD_OR_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12198 case MVT::nxv4i32: return fastEmit_ISD_OR_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12199 case MVT::nxv2i64: return fastEmit_ISD_OR_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12200 default: return Register();
12201 }
12202}
12203
12204// FastEmit functions for ISD::ROTR.
12205
12206Register fastEmit_ISD_ROTR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12207 if (RetVT.SimpleTy != MVT::i64)
12208 return Register();
12209 return fastEmitInst_rr(MachineInstOpcode: AArch64::RORVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12210}
12211
12212Register fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12213 switch (VT.SimpleTy) {
12214 case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_rr(RetVT, Op0, Op1);
12215 default: return Register();
12216 }
12217}
12218
12219// FastEmit functions for ISD::SADDSAT.
12220
12221Register fastEmit_ISD_SADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12222 if (RetVT.SimpleTy != MVT::v8i8)
12223 return Register();
12224 if ((Subtarget->isNeonAvailable())) {
12225 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12226 }
12227 return Register();
12228}
12229
12230Register fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12231 if (RetVT.SimpleTy != MVT::v16i8)
12232 return Register();
12233 if ((Subtarget->isNeonAvailable())) {
12234 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12235 }
12236 return Register();
12237}
12238
12239Register fastEmit_ISD_SADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12240 if (RetVT.SimpleTy != MVT::v4i16)
12241 return Register();
12242 if ((Subtarget->isNeonAvailable())) {
12243 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12244 }
12245 return Register();
12246}
12247
12248Register fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12249 if (RetVT.SimpleTy != MVT::v8i16)
12250 return Register();
12251 if ((Subtarget->isNeonAvailable())) {
12252 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12253 }
12254 return Register();
12255}
12256
12257Register fastEmit_ISD_SADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12258 if (RetVT.SimpleTy != MVT::v2i32)
12259 return Register();
12260 if ((Subtarget->isNeonAvailable())) {
12261 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12262 }
12263 return Register();
12264}
12265
12266Register fastEmit_ISD_SADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12267 if (RetVT.SimpleTy != MVT::v4i32)
12268 return Register();
12269 if ((Subtarget->isNeonAvailable())) {
12270 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12271 }
12272 return Register();
12273}
12274
12275Register fastEmit_ISD_SADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12276 if (RetVT.SimpleTy != MVT::v1i64)
12277 return Register();
12278 if ((Subtarget->isNeonAvailable())) {
12279 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
12280 }
12281 return Register();
12282}
12283
12284Register fastEmit_ISD_SADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12285 if (RetVT.SimpleTy != MVT::v2i64)
12286 return Register();
12287 if ((Subtarget->isNeonAvailable())) {
12288 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12289 }
12290 return Register();
12291}
12292
12293Register fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12294 if (RetVT.SimpleTy != MVT::nxv16i8)
12295 return Register();
12296 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12297 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
12298 }
12299 return Register();
12300}
12301
12302Register fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12303 if (RetVT.SimpleTy != MVT::nxv8i16)
12304 return Register();
12305 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12306 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
12307 }
12308 return Register();
12309}
12310
12311Register fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12312 if (RetVT.SimpleTy != MVT::nxv4i32)
12313 return Register();
12314 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12315 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
12316 }
12317 return Register();
12318}
12319
12320Register fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12321 if (RetVT.SimpleTy != MVT::nxv2i64)
12322 return Register();
12323 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12324 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
12325 }
12326 return Register();
12327}
12328
12329Register fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12330 switch (VT.SimpleTy) {
12331 case MVT::v8i8: return fastEmit_ISD_SADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
12332 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12333 case MVT::v4i16: return fastEmit_ISD_SADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
12334 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12335 case MVT::v2i32: return fastEmit_ISD_SADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
12336 case MVT::v4i32: return fastEmit_ISD_SADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
12337 case MVT::v1i64: return fastEmit_ISD_SADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
12338 case MVT::v2i64: return fastEmit_ISD_SADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
12339 case MVT::nxv16i8: return fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12340 case MVT::nxv8i16: return fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12341 case MVT::nxv4i32: return fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12342 case MVT::nxv2i64: return fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12343 default: return Register();
12344 }
12345}
12346
12347// FastEmit functions for ISD::SDIV.
12348
12349Register fastEmit_ISD_SDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12350 if (RetVT.SimpleTy != MVT::i32)
12351 return Register();
12352 return fastEmitInst_rr(MachineInstOpcode: AArch64::SDIVWr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12353}
12354
12355Register fastEmit_ISD_SDIV_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12356 if (RetVT.SimpleTy != MVT::i64)
12357 return Register();
12358 return fastEmitInst_rr(MachineInstOpcode: AArch64::SDIVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12359}
12360
12361Register fastEmit_ISD_SDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12362 switch (VT.SimpleTy) {
12363 case MVT::i32: return fastEmit_ISD_SDIV_MVT_i32_rr(RetVT, Op0, Op1);
12364 case MVT::i64: return fastEmit_ISD_SDIV_MVT_i64_rr(RetVT, Op0, Op1);
12365 default: return Register();
12366 }
12367}
12368
12369// FastEmit functions for ISD::SHL.
12370
12371Register fastEmit_ISD_SHL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12372 if (RetVT.SimpleTy != MVT::i64)
12373 return Register();
12374 return fastEmitInst_rr(MachineInstOpcode: AArch64::LSLVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12375}
12376
12377Register fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12378 switch (VT.SimpleTy) {
12379 case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_rr(RetVT, Op0, Op1);
12380 default: return Register();
12381 }
12382}
12383
12384// FastEmit functions for ISD::SMAX.
12385
12386Register fastEmit_ISD_SMAX_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12387 if (RetVT.SimpleTy != MVT::i32)
12388 return Register();
12389 if ((Subtarget->hasCSSC())) {
12390 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12391 }
12392 return Register();
12393}
12394
12395Register fastEmit_ISD_SMAX_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12396 if (RetVT.SimpleTy != MVT::i64)
12397 return Register();
12398 if ((Subtarget->hasCSSC())) {
12399 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12400 }
12401 return Register();
12402}
12403
12404Register fastEmit_ISD_SMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12405 if (RetVT.SimpleTy != MVT::v8i8)
12406 return Register();
12407 if ((Subtarget->isNeonAvailable())) {
12408 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12409 }
12410 return Register();
12411}
12412
12413Register fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12414 if (RetVT.SimpleTy != MVT::v16i8)
12415 return Register();
12416 if ((Subtarget->isNeonAvailable())) {
12417 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12418 }
12419 return Register();
12420}
12421
12422Register fastEmit_ISD_SMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12423 if (RetVT.SimpleTy != MVT::v4i16)
12424 return Register();
12425 if ((Subtarget->isNeonAvailable())) {
12426 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12427 }
12428 return Register();
12429}
12430
12431Register fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12432 if (RetVT.SimpleTy != MVT::v8i16)
12433 return Register();
12434 if ((Subtarget->isNeonAvailable())) {
12435 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12436 }
12437 return Register();
12438}
12439
12440Register fastEmit_ISD_SMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12441 if (RetVT.SimpleTy != MVT::v2i32)
12442 return Register();
12443 if ((Subtarget->isNeonAvailable())) {
12444 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12445 }
12446 return Register();
12447}
12448
12449Register fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12450 if (RetVT.SimpleTy != MVT::v4i32)
12451 return Register();
12452 if ((Subtarget->isNeonAvailable())) {
12453 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12454 }
12455 return Register();
12456}
12457
12458Register fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12459 switch (VT.SimpleTy) {
12460 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_rr(RetVT, Op0, Op1);
12461 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_rr(RetVT, Op0, Op1);
12462 case MVT::v8i8: return fastEmit_ISD_SMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
12463 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
12464 case MVT::v4i16: return fastEmit_ISD_SMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
12465 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
12466 case MVT::v2i32: return fastEmit_ISD_SMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
12467 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
12468 default: return Register();
12469 }
12470}
12471
12472// FastEmit functions for ISD::SMIN.
12473
12474Register fastEmit_ISD_SMIN_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12475 if (RetVT.SimpleTy != MVT::i32)
12476 return Register();
12477 if ((Subtarget->hasCSSC())) {
12478 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12479 }
12480 return Register();
12481}
12482
12483Register fastEmit_ISD_SMIN_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12484 if (RetVT.SimpleTy != MVT::i64)
12485 return Register();
12486 if ((Subtarget->hasCSSC())) {
12487 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12488 }
12489 return Register();
12490}
12491
12492Register fastEmit_ISD_SMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12493 if (RetVT.SimpleTy != MVT::v8i8)
12494 return Register();
12495 if ((Subtarget->isNeonAvailable())) {
12496 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12497 }
12498 return Register();
12499}
12500
12501Register fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12502 if (RetVT.SimpleTy != MVT::v16i8)
12503 return Register();
12504 if ((Subtarget->isNeonAvailable())) {
12505 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12506 }
12507 return Register();
12508}
12509
12510Register fastEmit_ISD_SMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12511 if (RetVT.SimpleTy != MVT::v4i16)
12512 return Register();
12513 if ((Subtarget->isNeonAvailable())) {
12514 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12515 }
12516 return Register();
12517}
12518
12519Register fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12520 if (RetVT.SimpleTy != MVT::v8i16)
12521 return Register();
12522 if ((Subtarget->isNeonAvailable())) {
12523 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12524 }
12525 return Register();
12526}
12527
12528Register fastEmit_ISD_SMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12529 if (RetVT.SimpleTy != MVT::v2i32)
12530 return Register();
12531 if ((Subtarget->isNeonAvailable())) {
12532 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12533 }
12534 return Register();
12535}
12536
12537Register fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12538 if (RetVT.SimpleTy != MVT::v4i32)
12539 return Register();
12540 if ((Subtarget->isNeonAvailable())) {
12541 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12542 }
12543 return Register();
12544}
12545
12546Register fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12547 switch (VT.SimpleTy) {
12548 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_rr(RetVT, Op0, Op1);
12549 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_rr(RetVT, Op0, Op1);
12550 case MVT::v8i8: return fastEmit_ISD_SMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
12551 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
12552 case MVT::v4i16: return fastEmit_ISD_SMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
12553 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
12554 case MVT::v2i32: return fastEmit_ISD_SMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
12555 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
12556 default: return Register();
12557 }
12558}
12559
12560// FastEmit functions for ISD::SRA.
12561
12562Register fastEmit_ISD_SRA_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12563 if (RetVT.SimpleTy != MVT::i64)
12564 return Register();
12565 return fastEmitInst_rr(MachineInstOpcode: AArch64::ASRVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12566}
12567
12568Register fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12569 switch (VT.SimpleTy) {
12570 case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_rr(RetVT, Op0, Op1);
12571 default: return Register();
12572 }
12573}
12574
12575// FastEmit functions for ISD::SRL.
12576
12577Register fastEmit_ISD_SRL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12578 if (RetVT.SimpleTy != MVT::i64)
12579 return Register();
12580 return fastEmitInst_rr(MachineInstOpcode: AArch64::LSRVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12581}
12582
12583Register fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12584 switch (VT.SimpleTy) {
12585 case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_rr(RetVT, Op0, Op1);
12586 default: return Register();
12587 }
12588}
12589
12590// FastEmit functions for ISD::SSUBSAT.
12591
12592Register fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12593 if (RetVT.SimpleTy != MVT::v8i8)
12594 return Register();
12595 if ((Subtarget->isNeonAvailable())) {
12596 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12597 }
12598 return Register();
12599}
12600
12601Register fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12602 if (RetVT.SimpleTy != MVT::v16i8)
12603 return Register();
12604 if ((Subtarget->isNeonAvailable())) {
12605 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12606 }
12607 return Register();
12608}
12609
12610Register fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12611 if (RetVT.SimpleTy != MVT::v4i16)
12612 return Register();
12613 if ((Subtarget->isNeonAvailable())) {
12614 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12615 }
12616 return Register();
12617}
12618
12619Register fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12620 if (RetVT.SimpleTy != MVT::v8i16)
12621 return Register();
12622 if ((Subtarget->isNeonAvailable())) {
12623 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12624 }
12625 return Register();
12626}
12627
12628Register fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12629 if (RetVT.SimpleTy != MVT::v2i32)
12630 return Register();
12631 if ((Subtarget->isNeonAvailable())) {
12632 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12633 }
12634 return Register();
12635}
12636
12637Register fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12638 if (RetVT.SimpleTy != MVT::v4i32)
12639 return Register();
12640 if ((Subtarget->isNeonAvailable())) {
12641 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12642 }
12643 return Register();
12644}
12645
12646Register fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12647 if (RetVT.SimpleTy != MVT::v1i64)
12648 return Register();
12649 if ((Subtarget->isNeonAvailable())) {
12650 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
12651 }
12652 return Register();
12653}
12654
12655Register fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12656 if (RetVT.SimpleTy != MVT::v2i64)
12657 return Register();
12658 if ((Subtarget->isNeonAvailable())) {
12659 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12660 }
12661 return Register();
12662}
12663
12664Register fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12665 if (RetVT.SimpleTy != MVT::nxv16i8)
12666 return Register();
12667 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12668 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
12669 }
12670 return Register();
12671}
12672
12673Register fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12674 if (RetVT.SimpleTy != MVT::nxv8i16)
12675 return Register();
12676 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12677 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
12678 }
12679 return Register();
12680}
12681
12682Register fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12683 if (RetVT.SimpleTy != MVT::nxv4i32)
12684 return Register();
12685 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12686 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
12687 }
12688 return Register();
12689}
12690
12691Register fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12692 if (RetVT.SimpleTy != MVT::nxv2i64)
12693 return Register();
12694 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12695 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
12696 }
12697 return Register();
12698}
12699
12700Register fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12701 switch (VT.SimpleTy) {
12702 case MVT::v8i8: return fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
12703 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12704 case MVT::v4i16: return fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
12705 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12706 case MVT::v2i32: return fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
12707 case MVT::v4i32: return fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
12708 case MVT::v1i64: return fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
12709 case MVT::v2i64: return fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
12710 case MVT::nxv16i8: return fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12711 case MVT::nxv8i16: return fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12712 case MVT::nxv4i32: return fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12713 case MVT::nxv2i64: return fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12714 default: return Register();
12715 }
12716}
12717
12718// FastEmit functions for ISD::STRICT_FADD.
12719
12720Register fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12721 if (RetVT.SimpleTy != MVT::f16)
12722 return Register();
12723 if ((Subtarget->hasFullFP16())) {
12724 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12725 }
12726 return Register();
12727}
12728
12729Register fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12730 if (RetVT.SimpleTy != MVT::f32)
12731 return Register();
12732 if ((Subtarget->hasFPARMv8())) {
12733 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12734 }
12735 return Register();
12736}
12737
12738Register fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12739 if (RetVT.SimpleTy != MVT::f64)
12740 return Register();
12741 if ((Subtarget->hasFPARMv8())) {
12742 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12743 }
12744 return Register();
12745}
12746
12747Register fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12748 if (RetVT.SimpleTy != MVT::v4f16)
12749 return Register();
12750 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12751 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12752 }
12753 return Register();
12754}
12755
12756Register fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12757 if (RetVT.SimpleTy != MVT::v8f16)
12758 return Register();
12759 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12760 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12761 }
12762 return Register();
12763}
12764
12765Register fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12766 if (RetVT.SimpleTy != MVT::v2f32)
12767 return Register();
12768 if ((Subtarget->isNeonAvailable())) {
12769 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12770 }
12771 return Register();
12772}
12773
12774Register fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12775 if (RetVT.SimpleTy != MVT::v4f32)
12776 return Register();
12777 if ((Subtarget->isNeonAvailable())) {
12778 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12779 }
12780 return Register();
12781}
12782
12783Register fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12784 if (RetVT.SimpleTy != MVT::v2f64)
12785 return Register();
12786 if ((Subtarget->isNeonAvailable())) {
12787 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12788 }
12789 return Register();
12790}
12791
12792Register fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12793 switch (VT.SimpleTy) {
12794 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1);
12795 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1);
12796 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1);
12797 case MVT::v4f16: return fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
12798 case MVT::v8f16: return fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
12799 case MVT::v2f32: return fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
12800 case MVT::v4f32: return fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
12801 case MVT::v2f64: return fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
12802 default: return Register();
12803 }
12804}
12805
12806// FastEmit functions for ISD::STRICT_FDIV.
12807
12808Register fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12809 if (RetVT.SimpleTy != MVT::f16)
12810 return Register();
12811 if ((Subtarget->hasFullFP16())) {
12812 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12813 }
12814 return Register();
12815}
12816
12817Register fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12818 if (RetVT.SimpleTy != MVT::f32)
12819 return Register();
12820 if ((Subtarget->hasFPARMv8())) {
12821 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12822 }
12823 return Register();
12824}
12825
12826Register fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12827 if (RetVT.SimpleTy != MVT::f64)
12828 return Register();
12829 if ((Subtarget->hasFPARMv8())) {
12830 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12831 }
12832 return Register();
12833}
12834
12835Register fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12836 if (RetVT.SimpleTy != MVT::v4f16)
12837 return Register();
12838 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12839 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12840 }
12841 return Register();
12842}
12843
12844Register fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12845 if (RetVT.SimpleTy != MVT::v8f16)
12846 return Register();
12847 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12848 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12849 }
12850 return Register();
12851}
12852
12853Register fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12854 if (RetVT.SimpleTy != MVT::v2f32)
12855 return Register();
12856 if ((Subtarget->isNeonAvailable())) {
12857 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12858 }
12859 return Register();
12860}
12861
12862Register fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12863 if (RetVT.SimpleTy != MVT::v4f32)
12864 return Register();
12865 if ((Subtarget->isNeonAvailable())) {
12866 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12867 }
12868 return Register();
12869}
12870
12871Register fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12872 if (RetVT.SimpleTy != MVT::v2f64)
12873 return Register();
12874 if ((Subtarget->isNeonAvailable())) {
12875 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12876 }
12877 return Register();
12878}
12879
12880Register fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12881 switch (VT.SimpleTy) {
12882 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
12883 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
12884 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
12885 case MVT::v4f16: return fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1);
12886 case MVT::v8f16: return fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
12887 case MVT::v2f32: return fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1);
12888 case MVT::v4f32: return fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
12889 case MVT::v2f64: return fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
12890 default: return Register();
12891 }
12892}
12893
12894// FastEmit functions for ISD::STRICT_FMAXIMUM.
12895
12896Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12897 if (RetVT.SimpleTy != MVT::f16)
12898 return Register();
12899 if ((Subtarget->hasFullFP16())) {
12900 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12901 }
12902 return Register();
12903}
12904
12905Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12906 if (RetVT.SimpleTy != MVT::f32)
12907 return Register();
12908 if ((Subtarget->hasFPARMv8())) {
12909 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12910 }
12911 return Register();
12912}
12913
12914Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12915 if (RetVT.SimpleTy != MVT::f64)
12916 return Register();
12917 if ((Subtarget->hasFPARMv8())) {
12918 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12919 }
12920 return Register();
12921}
12922
12923Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12924 if (RetVT.SimpleTy != MVT::v4f16)
12925 return Register();
12926 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12927 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12928 }
12929 return Register();
12930}
12931
12932Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12933 if (RetVT.SimpleTy != MVT::v8f16)
12934 return Register();
12935 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12936 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12937 }
12938 return Register();
12939}
12940
12941Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12942 if (RetVT.SimpleTy != MVT::v2f32)
12943 return Register();
12944 if ((Subtarget->isNeonAvailable())) {
12945 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12946 }
12947 return Register();
12948}
12949
12950Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12951 if (RetVT.SimpleTy != MVT::v4f32)
12952 return Register();
12953 if ((Subtarget->isNeonAvailable())) {
12954 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12955 }
12956 return Register();
12957}
12958
12959Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12960 if (RetVT.SimpleTy != MVT::v2f64)
12961 return Register();
12962 if ((Subtarget->isNeonAvailable())) {
12963 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12964 }
12965 return Register();
12966}
12967
12968Register fastEmit_ISD_STRICT_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12969 switch (VT.SimpleTy) {
12970 case MVT::f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1);
12971 case MVT::f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1);
12972 case MVT::f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1);
12973 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
12974 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
12975 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
12976 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
12977 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
12978 default: return Register();
12979 }
12980}
12981
12982// FastEmit functions for ISD::STRICT_FMAXNUM.
12983
12984Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12985 if (RetVT.SimpleTy != MVT::f16)
12986 return Register();
12987 if ((Subtarget->hasFullFP16())) {
12988 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12989 }
12990 return Register();
12991}
12992
12993Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12994 if (RetVT.SimpleTy != MVT::f32)
12995 return Register();
12996 if ((Subtarget->hasFPARMv8())) {
12997 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12998 }
12999 return Register();
13000}
13001
13002Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13003 if (RetVT.SimpleTy != MVT::f64)
13004 return Register();
13005 if ((Subtarget->hasFPARMv8())) {
13006 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13007 }
13008 return Register();
13009}
13010
13011Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13012 if (RetVT.SimpleTy != MVT::v4f16)
13013 return Register();
13014 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13015 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13016 }
13017 return Register();
13018}
13019
13020Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13021 if (RetVT.SimpleTy != MVT::v8f16)
13022 return Register();
13023 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13024 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13025 }
13026 return Register();
13027}
13028
13029Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13030 if (RetVT.SimpleTy != MVT::v2f32)
13031 return Register();
13032 if ((Subtarget->isNeonAvailable())) {
13033 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13034 }
13035 return Register();
13036}
13037
13038Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13039 if (RetVT.SimpleTy != MVT::v4f32)
13040 return Register();
13041 if ((Subtarget->isNeonAvailable())) {
13042 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13043 }
13044 return Register();
13045}
13046
13047Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13048 if (RetVT.SimpleTy != MVT::v2f64)
13049 return Register();
13050 if ((Subtarget->isNeonAvailable())) {
13051 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13052 }
13053 return Register();
13054}
13055
13056Register fastEmit_ISD_STRICT_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13057 switch (VT.SimpleTy) {
13058 case MVT::f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
13059 case MVT::f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
13060 case MVT::f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
13061 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13062 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13063 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13064 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13065 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13066 default: return Register();
13067 }
13068}
13069
13070// FastEmit functions for ISD::STRICT_FMINIMUM.
13071
13072Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13073 if (RetVT.SimpleTy != MVT::f16)
13074 return Register();
13075 if ((Subtarget->hasFullFP16())) {
13076 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13077 }
13078 return Register();
13079}
13080
13081Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13082 if (RetVT.SimpleTy != MVT::f32)
13083 return Register();
13084 if ((Subtarget->hasFPARMv8())) {
13085 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13086 }
13087 return Register();
13088}
13089
13090Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13091 if (RetVT.SimpleTy != MVT::f64)
13092 return Register();
13093 if ((Subtarget->hasFPARMv8())) {
13094 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13095 }
13096 return Register();
13097}
13098
13099Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13100 if (RetVT.SimpleTy != MVT::v4f16)
13101 return Register();
13102 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13103 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13104 }
13105 return Register();
13106}
13107
13108Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13109 if (RetVT.SimpleTy != MVT::v8f16)
13110 return Register();
13111 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13112 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13113 }
13114 return Register();
13115}
13116
13117Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13118 if (RetVT.SimpleTy != MVT::v2f32)
13119 return Register();
13120 if ((Subtarget->isNeonAvailable())) {
13121 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13122 }
13123 return Register();
13124}
13125
13126Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13127 if (RetVT.SimpleTy != MVT::v4f32)
13128 return Register();
13129 if ((Subtarget->isNeonAvailable())) {
13130 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13131 }
13132 return Register();
13133}
13134
13135Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13136 if (RetVT.SimpleTy != MVT::v2f64)
13137 return Register();
13138 if ((Subtarget->isNeonAvailable())) {
13139 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13140 }
13141 return Register();
13142}
13143
13144Register fastEmit_ISD_STRICT_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13145 switch (VT.SimpleTy) {
13146 case MVT::f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1);
13147 case MVT::f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1);
13148 case MVT::f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1);
13149 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13150 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13151 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13152 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13153 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13154 default: return Register();
13155 }
13156}
13157
13158// FastEmit functions for ISD::STRICT_FMINNUM.
13159
13160Register fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13161 if (RetVT.SimpleTy != MVT::f16)
13162 return Register();
13163 if ((Subtarget->hasFullFP16())) {
13164 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13165 }
13166 return Register();
13167}
13168
13169Register fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13170 if (RetVT.SimpleTy != MVT::f32)
13171 return Register();
13172 if ((Subtarget->hasFPARMv8())) {
13173 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13174 }
13175 return Register();
13176}
13177
13178Register fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13179 if (RetVT.SimpleTy != MVT::f64)
13180 return Register();
13181 if ((Subtarget->hasFPARMv8())) {
13182 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13183 }
13184 return Register();
13185}
13186
13187Register fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13188 if (RetVT.SimpleTy != MVT::v4f16)
13189 return Register();
13190 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13191 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13192 }
13193 return Register();
13194}
13195
13196Register fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13197 if (RetVT.SimpleTy != MVT::v8f16)
13198 return Register();
13199 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13200 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13201 }
13202 return Register();
13203}
13204
13205Register fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13206 if (RetVT.SimpleTy != MVT::v2f32)
13207 return Register();
13208 if ((Subtarget->isNeonAvailable())) {
13209 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13210 }
13211 return Register();
13212}
13213
13214Register fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13215 if (RetVT.SimpleTy != MVT::v4f32)
13216 return Register();
13217 if ((Subtarget->isNeonAvailable())) {
13218 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13219 }
13220 return Register();
13221}
13222
13223Register fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13224 if (RetVT.SimpleTy != MVT::v2f64)
13225 return Register();
13226 if ((Subtarget->isNeonAvailable())) {
13227 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13228 }
13229 return Register();
13230}
13231
13232Register fastEmit_ISD_STRICT_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13233 switch (VT.SimpleTy) {
13234 case MVT::f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
13235 case MVT::f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
13236 case MVT::f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
13237 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13238 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13239 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13240 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13241 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13242 default: return Register();
13243 }
13244}
13245
13246// FastEmit functions for ISD::STRICT_FMUL.
13247
13248Register fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13249 if (RetVT.SimpleTy != MVT::f16)
13250 return Register();
13251 if ((Subtarget->hasFullFP16())) {
13252 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13253 }
13254 return Register();
13255}
13256
13257Register fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13258 if (RetVT.SimpleTy != MVT::f32)
13259 return Register();
13260 if ((Subtarget->hasFPARMv8())) {
13261 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13262 }
13263 return Register();
13264}
13265
13266Register fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13267 if (RetVT.SimpleTy != MVT::f64)
13268 return Register();
13269 if ((Subtarget->hasFPARMv8())) {
13270 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13271 }
13272 return Register();
13273}
13274
13275Register fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13276 if (RetVT.SimpleTy != MVT::v4f16)
13277 return Register();
13278 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13279 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13280 }
13281 return Register();
13282}
13283
13284Register fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13285 if (RetVT.SimpleTy != MVT::v8f16)
13286 return Register();
13287 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13288 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13289 }
13290 return Register();
13291}
13292
13293Register fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13294 if (RetVT.SimpleTy != MVT::v2f32)
13295 return Register();
13296 if ((Subtarget->isNeonAvailable())) {
13297 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13298 }
13299 return Register();
13300}
13301
13302Register fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13303 if (RetVT.SimpleTy != MVT::v4f32)
13304 return Register();
13305 if ((Subtarget->isNeonAvailable())) {
13306 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13307 }
13308 return Register();
13309}
13310
13311Register fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13312 if (RetVT.SimpleTy != MVT::v2f64)
13313 return Register();
13314 if ((Subtarget->isNeonAvailable())) {
13315 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13316 }
13317 return Register();
13318}
13319
13320Register fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13321 switch (VT.SimpleTy) {
13322 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
13323 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
13324 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
13325 case MVT::v4f16: return fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
13326 case MVT::v8f16: return fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
13327 case MVT::v2f32: return fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
13328 case MVT::v4f32: return fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
13329 case MVT::v2f64: return fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
13330 default: return Register();
13331 }
13332}
13333
13334// FastEmit functions for ISD::STRICT_FSUB.
13335
13336Register fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13337 if (RetVT.SimpleTy != MVT::f16)
13338 return Register();
13339 if ((Subtarget->hasFullFP16())) {
13340 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13341 }
13342 return Register();
13343}
13344
13345Register fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13346 if (RetVT.SimpleTy != MVT::f32)
13347 return Register();
13348 if ((Subtarget->hasFPARMv8())) {
13349 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13350 }
13351 return Register();
13352}
13353
13354Register fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13355 if (RetVT.SimpleTy != MVT::f64)
13356 return Register();
13357 if ((Subtarget->hasFPARMv8())) {
13358 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13359 }
13360 return Register();
13361}
13362
13363Register fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13364 if (RetVT.SimpleTy != MVT::v4f16)
13365 return Register();
13366 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13367 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13368 }
13369 return Register();
13370}
13371
13372Register fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13373 if (RetVT.SimpleTy != MVT::v8f16)
13374 return Register();
13375 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13376 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13377 }
13378 return Register();
13379}
13380
13381Register fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13382 if (RetVT.SimpleTy != MVT::v2f32)
13383 return Register();
13384 if ((Subtarget->isNeonAvailable())) {
13385 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13386 }
13387 return Register();
13388}
13389
13390Register fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13391 if (RetVT.SimpleTy != MVT::v4f32)
13392 return Register();
13393 if ((Subtarget->isNeonAvailable())) {
13394 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13395 }
13396 return Register();
13397}
13398
13399Register fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13400 if (RetVT.SimpleTy != MVT::v2f64)
13401 return Register();
13402 if ((Subtarget->isNeonAvailable())) {
13403 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13404 }
13405 return Register();
13406}
13407
13408Register fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13409 switch (VT.SimpleTy) {
13410 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
13411 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
13412 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
13413 case MVT::v4f16: return fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
13414 case MVT::v8f16: return fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
13415 case MVT::v2f32: return fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
13416 case MVT::v4f32: return fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
13417 case MVT::v2f64: return fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
13418 default: return Register();
13419 }
13420}
13421
13422// FastEmit functions for ISD::SUB.
13423
13424Register fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13425 if (RetVT.SimpleTy != MVT::i32)
13426 return Register();
13427 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBSWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13428}
13429
13430Register fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13431 if (RetVT.SimpleTy != MVT::i64)
13432 return Register();
13433 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBSXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13434}
13435
13436Register fastEmit_ISD_SUB_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13437 if (RetVT.SimpleTy != MVT::v8i8)
13438 return Register();
13439 if ((Subtarget->isNeonAvailable())) {
13440 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13441 }
13442 return Register();
13443}
13444
13445Register fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13446 if (RetVT.SimpleTy != MVT::v16i8)
13447 return Register();
13448 if ((Subtarget->isNeonAvailable())) {
13449 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13450 }
13451 return Register();
13452}
13453
13454Register fastEmit_ISD_SUB_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13455 if (RetVT.SimpleTy != MVT::v4i16)
13456 return Register();
13457 if ((Subtarget->isNeonAvailable())) {
13458 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13459 }
13460 return Register();
13461}
13462
13463Register fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13464 if (RetVT.SimpleTy != MVT::v8i16)
13465 return Register();
13466 if ((Subtarget->isNeonAvailable())) {
13467 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13468 }
13469 return Register();
13470}
13471
13472Register fastEmit_ISD_SUB_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13473 if (RetVT.SimpleTy != MVT::v2i32)
13474 return Register();
13475 if ((Subtarget->isNeonAvailable())) {
13476 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13477 }
13478 return Register();
13479}
13480
13481Register fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13482 if (RetVT.SimpleTy != MVT::v4i32)
13483 return Register();
13484 if ((Subtarget->isNeonAvailable())) {
13485 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13486 }
13487 return Register();
13488}
13489
13490Register fastEmit_ISD_SUB_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13491 if (RetVT.SimpleTy != MVT::v1i64)
13492 return Register();
13493 if ((Subtarget->isNeonAvailable())) {
13494 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13495 }
13496 return Register();
13497}
13498
13499Register fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13500 if (RetVT.SimpleTy != MVT::v2i64)
13501 return Register();
13502 if ((Subtarget->isNeonAvailable())) {
13503 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13504 }
13505 return Register();
13506}
13507
13508Register fastEmit_ISD_SUB_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13509 if (RetVT.SimpleTy != MVT::nxv16i8)
13510 return Register();
13511 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13512 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13513 }
13514 return Register();
13515}
13516
13517Register fastEmit_ISD_SUB_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13518 if (RetVT.SimpleTy != MVT::nxv8i16)
13519 return Register();
13520 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13521 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13522 }
13523 return Register();
13524}
13525
13526Register fastEmit_ISD_SUB_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13527 if (RetVT.SimpleTy != MVT::nxv4i32)
13528 return Register();
13529 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13530 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13531 }
13532 return Register();
13533}
13534
13535Register fastEmit_ISD_SUB_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13536 if (RetVT.SimpleTy != MVT::nxv2i64)
13537 return Register();
13538 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13539 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13540 }
13541 return Register();
13542}
13543
13544Register fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13545 switch (VT.SimpleTy) {
13546 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
13547 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op1);
13548 case MVT::v8i8: return fastEmit_ISD_SUB_MVT_v8i8_rr(RetVT, Op0, Op1);
13549 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
13550 case MVT::v4i16: return fastEmit_ISD_SUB_MVT_v4i16_rr(RetVT, Op0, Op1);
13551 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
13552 case MVT::v2i32: return fastEmit_ISD_SUB_MVT_v2i32_rr(RetVT, Op0, Op1);
13553 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
13554 case MVT::v1i64: return fastEmit_ISD_SUB_MVT_v1i64_rr(RetVT, Op0, Op1);
13555 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
13556 case MVT::nxv16i8: return fastEmit_ISD_SUB_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13557 case MVT::nxv8i16: return fastEmit_ISD_SUB_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13558 case MVT::nxv4i32: return fastEmit_ISD_SUB_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13559 case MVT::nxv2i64: return fastEmit_ISD_SUB_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13560 default: return Register();
13561 }
13562}
13563
13564// FastEmit functions for ISD::UADDSAT.
13565
13566Register fastEmit_ISD_UADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13567 if (RetVT.SimpleTy != MVT::v8i8)
13568 return Register();
13569 if ((Subtarget->isNeonAvailable())) {
13570 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13571 }
13572 return Register();
13573}
13574
13575Register fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13576 if (RetVT.SimpleTy != MVT::v16i8)
13577 return Register();
13578 if ((Subtarget->isNeonAvailable())) {
13579 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13580 }
13581 return Register();
13582}
13583
13584Register fastEmit_ISD_UADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13585 if (RetVT.SimpleTy != MVT::v4i16)
13586 return Register();
13587 if ((Subtarget->isNeonAvailable())) {
13588 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13589 }
13590 return Register();
13591}
13592
13593Register fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13594 if (RetVT.SimpleTy != MVT::v8i16)
13595 return Register();
13596 if ((Subtarget->isNeonAvailable())) {
13597 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13598 }
13599 return Register();
13600}
13601
13602Register fastEmit_ISD_UADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13603 if (RetVT.SimpleTy != MVT::v2i32)
13604 return Register();
13605 if ((Subtarget->isNeonAvailable())) {
13606 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13607 }
13608 return Register();
13609}
13610
13611Register fastEmit_ISD_UADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13612 if (RetVT.SimpleTy != MVT::v4i32)
13613 return Register();
13614 if ((Subtarget->isNeonAvailable())) {
13615 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13616 }
13617 return Register();
13618}
13619
13620Register fastEmit_ISD_UADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13621 if (RetVT.SimpleTy != MVT::v1i64)
13622 return Register();
13623 if ((Subtarget->isNeonAvailable())) {
13624 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13625 }
13626 return Register();
13627}
13628
13629Register fastEmit_ISD_UADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13630 if (RetVT.SimpleTy != MVT::v2i64)
13631 return Register();
13632 if ((Subtarget->isNeonAvailable())) {
13633 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13634 }
13635 return Register();
13636}
13637
13638Register fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13639 if (RetVT.SimpleTy != MVT::nxv16i8)
13640 return Register();
13641 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13642 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13643 }
13644 return Register();
13645}
13646
13647Register fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13648 if (RetVT.SimpleTy != MVT::nxv8i16)
13649 return Register();
13650 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13651 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13652 }
13653 return Register();
13654}
13655
13656Register fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13657 if (RetVT.SimpleTy != MVT::nxv4i32)
13658 return Register();
13659 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13660 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13661 }
13662 return Register();
13663}
13664
13665Register fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13666 if (RetVT.SimpleTy != MVT::nxv2i64)
13667 return Register();
13668 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13669 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13670 }
13671 return Register();
13672}
13673
13674Register fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13675 switch (VT.SimpleTy) {
13676 case MVT::v8i8: return fastEmit_ISD_UADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
13677 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13678 case MVT::v4i16: return fastEmit_ISD_UADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
13679 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13680 case MVT::v2i32: return fastEmit_ISD_UADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
13681 case MVT::v4i32: return fastEmit_ISD_UADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
13682 case MVT::v1i64: return fastEmit_ISD_UADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
13683 case MVT::v2i64: return fastEmit_ISD_UADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
13684 case MVT::nxv16i8: return fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13685 case MVT::nxv8i16: return fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13686 case MVT::nxv4i32: return fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13687 case MVT::nxv2i64: return fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13688 default: return Register();
13689 }
13690}
13691
13692// FastEmit functions for ISD::UDIV.
13693
13694Register fastEmit_ISD_UDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13695 if (RetVT.SimpleTy != MVT::i32)
13696 return Register();
13697 return fastEmitInst_rr(MachineInstOpcode: AArch64::UDIVWr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13698}
13699
13700Register fastEmit_ISD_UDIV_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13701 if (RetVT.SimpleTy != MVT::i64)
13702 return Register();
13703 return fastEmitInst_rr(MachineInstOpcode: AArch64::UDIVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13704}
13705
13706Register fastEmit_ISD_UDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13707 switch (VT.SimpleTy) {
13708 case MVT::i32: return fastEmit_ISD_UDIV_MVT_i32_rr(RetVT, Op0, Op1);
13709 case MVT::i64: return fastEmit_ISD_UDIV_MVT_i64_rr(RetVT, Op0, Op1);
13710 default: return Register();
13711 }
13712}
13713
13714// FastEmit functions for ISD::UMAX.
13715
13716Register fastEmit_ISD_UMAX_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13717 if (RetVT.SimpleTy != MVT::i32)
13718 return Register();
13719 if ((Subtarget->hasCSSC())) {
13720 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13721 }
13722 return Register();
13723}
13724
13725Register fastEmit_ISD_UMAX_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13726 if (RetVT.SimpleTy != MVT::i64)
13727 return Register();
13728 if ((Subtarget->hasCSSC())) {
13729 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13730 }
13731 return Register();
13732}
13733
13734Register fastEmit_ISD_UMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13735 if (RetVT.SimpleTy != MVT::v8i8)
13736 return Register();
13737 if ((Subtarget->isNeonAvailable())) {
13738 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13739 }
13740 return Register();
13741}
13742
13743Register fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13744 if (RetVT.SimpleTy != MVT::v16i8)
13745 return Register();
13746 if ((Subtarget->isNeonAvailable())) {
13747 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13748 }
13749 return Register();
13750}
13751
13752Register fastEmit_ISD_UMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13753 if (RetVT.SimpleTy != MVT::v4i16)
13754 return Register();
13755 if ((Subtarget->isNeonAvailable())) {
13756 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13757 }
13758 return Register();
13759}
13760
13761Register fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13762 if (RetVT.SimpleTy != MVT::v8i16)
13763 return Register();
13764 if ((Subtarget->isNeonAvailable())) {
13765 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13766 }
13767 return Register();
13768}
13769
13770Register fastEmit_ISD_UMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13771 if (RetVT.SimpleTy != MVT::v2i32)
13772 return Register();
13773 if ((Subtarget->isNeonAvailable())) {
13774 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13775 }
13776 return Register();
13777}
13778
13779Register fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13780 if (RetVT.SimpleTy != MVT::v4i32)
13781 return Register();
13782 if ((Subtarget->isNeonAvailable())) {
13783 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13784 }
13785 return Register();
13786}
13787
13788Register fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13789 switch (VT.SimpleTy) {
13790 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_rr(RetVT, Op0, Op1);
13791 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_rr(RetVT, Op0, Op1);
13792 case MVT::v8i8: return fastEmit_ISD_UMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
13793 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
13794 case MVT::v4i16: return fastEmit_ISD_UMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
13795 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
13796 case MVT::v2i32: return fastEmit_ISD_UMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
13797 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
13798 default: return Register();
13799 }
13800}
13801
13802// FastEmit functions for ISD::UMIN.
13803
13804Register fastEmit_ISD_UMIN_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13805 if (RetVT.SimpleTy != MVT::i32)
13806 return Register();
13807 if ((Subtarget->hasCSSC())) {
13808 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13809 }
13810 return Register();
13811}
13812
13813Register fastEmit_ISD_UMIN_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13814 if (RetVT.SimpleTy != MVT::i64)
13815 return Register();
13816 if ((Subtarget->hasCSSC())) {
13817 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13818 }
13819 return Register();
13820}
13821
13822Register fastEmit_ISD_UMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13823 if (RetVT.SimpleTy != MVT::v8i8)
13824 return Register();
13825 if ((Subtarget->isNeonAvailable())) {
13826 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13827 }
13828 return Register();
13829}
13830
13831Register fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13832 if (RetVT.SimpleTy != MVT::v16i8)
13833 return Register();
13834 if ((Subtarget->isNeonAvailable())) {
13835 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13836 }
13837 return Register();
13838}
13839
13840Register fastEmit_ISD_UMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13841 if (RetVT.SimpleTy != MVT::v4i16)
13842 return Register();
13843 if ((Subtarget->isNeonAvailable())) {
13844 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13845 }
13846 return Register();
13847}
13848
13849Register fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13850 if (RetVT.SimpleTy != MVT::v8i16)
13851 return Register();
13852 if ((Subtarget->isNeonAvailable())) {
13853 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13854 }
13855 return Register();
13856}
13857
13858Register fastEmit_ISD_UMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13859 if (RetVT.SimpleTy != MVT::v2i32)
13860 return Register();
13861 if ((Subtarget->isNeonAvailable())) {
13862 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13863 }
13864 return Register();
13865}
13866
13867Register fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13868 if (RetVT.SimpleTy != MVT::v4i32)
13869 return Register();
13870 if ((Subtarget->isNeonAvailable())) {
13871 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13872 }
13873 return Register();
13874}
13875
13876Register fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13877 switch (VT.SimpleTy) {
13878 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_rr(RetVT, Op0, Op1);
13879 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_rr(RetVT, Op0, Op1);
13880 case MVT::v8i8: return fastEmit_ISD_UMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
13881 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
13882 case MVT::v4i16: return fastEmit_ISD_UMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
13883 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
13884 case MVT::v2i32: return fastEmit_ISD_UMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
13885 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
13886 default: return Register();
13887 }
13888}
13889
13890// FastEmit functions for ISD::USUBSAT.
13891
13892Register fastEmit_ISD_USUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13893 if (RetVT.SimpleTy != MVT::v8i8)
13894 return Register();
13895 if ((Subtarget->isNeonAvailable())) {
13896 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13897 }
13898 return Register();
13899}
13900
13901Register fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13902 if (RetVT.SimpleTy != MVT::v16i8)
13903 return Register();
13904 if ((Subtarget->isNeonAvailable())) {
13905 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13906 }
13907 return Register();
13908}
13909
13910Register fastEmit_ISD_USUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13911 if (RetVT.SimpleTy != MVT::v4i16)
13912 return Register();
13913 if ((Subtarget->isNeonAvailable())) {
13914 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13915 }
13916 return Register();
13917}
13918
13919Register fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13920 if (RetVT.SimpleTy != MVT::v8i16)
13921 return Register();
13922 if ((Subtarget->isNeonAvailable())) {
13923 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13924 }
13925 return Register();
13926}
13927
13928Register fastEmit_ISD_USUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13929 if (RetVT.SimpleTy != MVT::v2i32)
13930 return Register();
13931 if ((Subtarget->isNeonAvailable())) {
13932 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13933 }
13934 return Register();
13935}
13936
13937Register fastEmit_ISD_USUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13938 if (RetVT.SimpleTy != MVT::v4i32)
13939 return Register();
13940 if ((Subtarget->isNeonAvailable())) {
13941 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13942 }
13943 return Register();
13944}
13945
13946Register fastEmit_ISD_USUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13947 if (RetVT.SimpleTy != MVT::v1i64)
13948 return Register();
13949 if ((Subtarget->isNeonAvailable())) {
13950 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13951 }
13952 return Register();
13953}
13954
13955Register fastEmit_ISD_USUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13956 if (RetVT.SimpleTy != MVT::v2i64)
13957 return Register();
13958 if ((Subtarget->isNeonAvailable())) {
13959 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13960 }
13961 return Register();
13962}
13963
13964Register fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13965 if (RetVT.SimpleTy != MVT::nxv16i8)
13966 return Register();
13967 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13968 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13969 }
13970 return Register();
13971}
13972
13973Register fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13974 if (RetVT.SimpleTy != MVT::nxv8i16)
13975 return Register();
13976 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13977 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13978 }
13979 return Register();
13980}
13981
13982Register fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13983 if (RetVT.SimpleTy != MVT::nxv4i32)
13984 return Register();
13985 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13986 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13987 }
13988 return Register();
13989}
13990
13991Register fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13992 if (RetVT.SimpleTy != MVT::nxv2i64)
13993 return Register();
13994 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13995 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13996 }
13997 return Register();
13998}
13999
14000Register fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14001 switch (VT.SimpleTy) {
14002 case MVT::v8i8: return fastEmit_ISD_USUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
14003 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
14004 case MVT::v4i16: return fastEmit_ISD_USUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
14005 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
14006 case MVT::v2i32: return fastEmit_ISD_USUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
14007 case MVT::v4i32: return fastEmit_ISD_USUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
14008 case MVT::v1i64: return fastEmit_ISD_USUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
14009 case MVT::v2i64: return fastEmit_ISD_USUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
14010 case MVT::nxv16i8: return fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
14011 case MVT::nxv8i16: return fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
14012 case MVT::nxv4i32: return fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
14013 case MVT::nxv2i64: return fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
14014 default: return Register();
14015 }
14016}
14017
14018// FastEmit functions for ISD::XOR.
14019
14020Register fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
14021 if (RetVT.SimpleTy != MVT::i32)
14022 return Register();
14023 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
14024}
14025
14026Register fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
14027 if (RetVT.SimpleTy != MVT::i64)
14028 return Register();
14029 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
14030}
14031
14032Register fastEmit_ISD_XOR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
14033 if (RetVT.SimpleTy != MVT::v8i8)
14034 return Register();
14035 if ((Subtarget->isNeonAvailable())) {
14036 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14037 }
14038 return Register();
14039}
14040
14041Register fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14042 if (RetVT.SimpleTy != MVT::v16i8)
14043 return Register();
14044 if ((Subtarget->isNeonAvailable())) {
14045 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14046 }
14047 return Register();
14048}
14049
14050Register fastEmit_ISD_XOR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
14051 if (RetVT.SimpleTy != MVT::v4i16)
14052 return Register();
14053 if ((Subtarget->isNeonAvailable())) {
14054 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14055 }
14056 return Register();
14057}
14058
14059Register fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14060 if (RetVT.SimpleTy != MVT::v8i16)
14061 return Register();
14062 if ((Subtarget->isNeonAvailable())) {
14063 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14064 }
14065 return Register();
14066}
14067
14068Register fastEmit_ISD_XOR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
14069 if (RetVT.SimpleTy != MVT::v2i32)
14070 return Register();
14071 if ((Subtarget->isNeonAvailable())) {
14072 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14073 }
14074 return Register();
14075}
14076
14077Register fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14078 if (RetVT.SimpleTy != MVT::v4i32)
14079 return Register();
14080 if ((Subtarget->isNeonAvailable())) {
14081 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14082 }
14083 return Register();
14084}
14085
14086Register fastEmit_ISD_XOR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
14087 if (RetVT.SimpleTy != MVT::v1i64)
14088 return Register();
14089 if ((Subtarget->isNeonAvailable())) {
14090 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14091 }
14092 return Register();
14093}
14094
14095Register fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14096 if (RetVT.SimpleTy != MVT::v2i64)
14097 return Register();
14098 if ((Subtarget->isNeonAvailable())) {
14099 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14100 }
14101 return Register();
14102}
14103
14104Register fastEmit_ISD_XOR_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14105 if (RetVT.SimpleTy != MVT::nxv16i8)
14106 return Register();
14107 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14108 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14109 }
14110 return Register();
14111}
14112
14113Register fastEmit_ISD_XOR_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14114 if (RetVT.SimpleTy != MVT::nxv8i16)
14115 return Register();
14116 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14117 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14118 }
14119 return Register();
14120}
14121
14122Register fastEmit_ISD_XOR_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14123 if (RetVT.SimpleTy != MVT::nxv4i32)
14124 return Register();
14125 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14126 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14127 }
14128 return Register();
14129}
14130
14131Register fastEmit_ISD_XOR_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14132 if (RetVT.SimpleTy != MVT::nxv2i64)
14133 return Register();
14134 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14135 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14136 }
14137 return Register();
14138}
14139
14140Register fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14141 switch (VT.SimpleTy) {
14142 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
14143 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op1);
14144 case MVT::v8i8: return fastEmit_ISD_XOR_MVT_v8i8_rr(RetVT, Op0, Op1);
14145 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
14146 case MVT::v4i16: return fastEmit_ISD_XOR_MVT_v4i16_rr(RetVT, Op0, Op1);
14147 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
14148 case MVT::v2i32: return fastEmit_ISD_XOR_MVT_v2i32_rr(RetVT, Op0, Op1);
14149 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
14150 case MVT::v1i64: return fastEmit_ISD_XOR_MVT_v1i64_rr(RetVT, Op0, Op1);
14151 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
14152 case MVT::nxv16i8: return fastEmit_ISD_XOR_MVT_nxv16i8_rr(RetVT, Op0, Op1);
14153 case MVT::nxv8i16: return fastEmit_ISD_XOR_MVT_nxv8i16_rr(RetVT, Op0, Op1);
14154 case MVT::nxv4i32: return fastEmit_ISD_XOR_MVT_nxv4i32_rr(RetVT, Op0, Op1);
14155 case MVT::nxv2i64: return fastEmit_ISD_XOR_MVT_nxv2i64_rr(RetVT, Op0, Op1);
14156 default: return Register();
14157 }
14158}
14159
14160// Top-level FastEmit function.
14161
14162Register fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, Register Op1) override {
14163 switch (Opcode) {
14164 case AArch64ISD::ADDHN: return fastEmit_AArch64ISD_ADDHN_rr(VT, RetVT, Op0, Op1);
14165 case AArch64ISD::ADDP: return fastEmit_AArch64ISD_ADDP_rr(VT, RetVT, Op0, Op1);
14166 case AArch64ISD::BIC: return fastEmit_AArch64ISD_BIC_rr(VT, RetVT, Op0, Op1);
14167 case AArch64ISD::CMTST: return fastEmit_AArch64ISD_CMTST_rr(VT, RetVT, Op0, Op1);
14168 case AArch64ISD::FCMEQ: return fastEmit_AArch64ISD_FCMEQ_rr(VT, RetVT, Op0, Op1);
14169 case AArch64ISD::FCMGE: return fastEmit_AArch64ISD_FCMGE_rr(VT, RetVT, Op0, Op1);
14170 case AArch64ISD::FCMGT: return fastEmit_AArch64ISD_FCMGT_rr(VT, RetVT, Op0, Op1);
14171 case AArch64ISD::FCMP: return fastEmit_AArch64ISD_FCMP_rr(VT, RetVT, Op0, Op1);
14172 case AArch64ISD::FRECPS: return fastEmit_AArch64ISD_FRECPS_rr(VT, RetVT, Op0, Op1);
14173 case AArch64ISD::FRSQRTS: return fastEmit_AArch64ISD_FRSQRTS_rr(VT, RetVT, Op0, Op1);
14174 case AArch64ISD::PMULL: return fastEmit_AArch64ISD_PMULL_rr(VT, RetVT, Op0, Op1);
14175 case AArch64ISD::PTEST: return fastEmit_AArch64ISD_PTEST_rr(VT, RetVT, Op0, Op1);
14176 case AArch64ISD::PTEST_ANY: return fastEmit_AArch64ISD_PTEST_ANY_rr(VT, RetVT, Op0, Op1);
14177 case AArch64ISD::PTEST_FIRST: return fastEmit_AArch64ISD_PTEST_FIRST_rr(VT, RetVT, Op0, Op1);
14178 case AArch64ISD::SMULL: return fastEmit_AArch64ISD_SMULL_rr(VT, RetVT, Op0, Op1);
14179 case AArch64ISD::SQADD: return fastEmit_AArch64ISD_SQADD_rr(VT, RetVT, Op0, Op1);
14180 case AArch64ISD::SQDMULH: return fastEmit_AArch64ISD_SQDMULH_rr(VT, RetVT, Op0, Op1);
14181 case AArch64ISD::SQDMULL: return fastEmit_AArch64ISD_SQDMULL_rr(VT, RetVT, Op0, Op1);
14182 case AArch64ISD::SQRDMULH: return fastEmit_AArch64ISD_SQRDMULH_rr(VT, RetVT, Op0, Op1);
14183 case AArch64ISD::SQRSHL: return fastEmit_AArch64ISD_SQRSHL_rr(VT, RetVT, Op0, Op1);
14184 case AArch64ISD::SQSHL: return fastEmit_AArch64ISD_SQSHL_rr(VT, RetVT, Op0, Op1);
14185 case AArch64ISD::SQSUB: return fastEmit_AArch64ISD_SQSUB_rr(VT, RetVT, Op0, Op1);
14186 case AArch64ISD::STRICT_FCMP: return fastEmit_AArch64ISD_STRICT_FCMP_rr(VT, RetVT, Op0, Op1);
14187 case AArch64ISD::STRICT_FCMPE: return fastEmit_AArch64ISD_STRICT_FCMPE_rr(VT, RetVT, Op0, Op1);
14188 case AArch64ISD::SUQADD: return fastEmit_AArch64ISD_SUQADD_rr(VT, RetVT, Op0, Op1);
14189 case AArch64ISD::TBL: return fastEmit_AArch64ISD_TBL_rr(VT, RetVT, Op0, Op1);
14190 case AArch64ISD::TRN1: return fastEmit_AArch64ISD_TRN1_rr(VT, RetVT, Op0, Op1);
14191 case AArch64ISD::TRN2: return fastEmit_AArch64ISD_TRN2_rr(VT, RetVT, Op0, Op1);
14192 case AArch64ISD::UMULL: return fastEmit_AArch64ISD_UMULL_rr(VT, RetVT, Op0, Op1);
14193 case AArch64ISD::UQADD: return fastEmit_AArch64ISD_UQADD_rr(VT, RetVT, Op0, Op1);
14194 case AArch64ISD::UQRSHL: return fastEmit_AArch64ISD_UQRSHL_rr(VT, RetVT, Op0, Op1);
14195 case AArch64ISD::UQSHL: return fastEmit_AArch64ISD_UQSHL_rr(VT, RetVT, Op0, Op1);
14196 case AArch64ISD::UQSUB: return fastEmit_AArch64ISD_UQSUB_rr(VT, RetVT, Op0, Op1);
14197 case AArch64ISD::USQADD: return fastEmit_AArch64ISD_USQADD_rr(VT, RetVT, Op0, Op1);
14198 case AArch64ISD::UZP1: return fastEmit_AArch64ISD_UZP1_rr(VT, RetVT, Op0, Op1);
14199 case AArch64ISD::UZP2: return fastEmit_AArch64ISD_UZP2_rr(VT, RetVT, Op0, Op1);
14200 case AArch64ISD::ZIP1: return fastEmit_AArch64ISD_ZIP1_rr(VT, RetVT, Op0, Op1);
14201 case AArch64ISD::ZIP2: return fastEmit_AArch64ISD_ZIP2_rr(VT, RetVT, Op0, Op1);
14202 case ISD::ABDS: return fastEmit_ISD_ABDS_rr(VT, RetVT, Op0, Op1);
14203 case ISD::ABDU: return fastEmit_ISD_ABDU_rr(VT, RetVT, Op0, Op1);
14204 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
14205 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
14206 case ISD::AVGCEILS: return fastEmit_ISD_AVGCEILS_rr(VT, RetVT, Op0, Op1);
14207 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
14208 case ISD::AVGFLOORS: return fastEmit_ISD_AVGFLOORS_rr(VT, RetVT, Op0, Op1);
14209 case ISD::AVGFLOORU: return fastEmit_ISD_AVGFLOORU_rr(VT, RetVT, Op0, Op1);
14210 case ISD::CLMUL: return fastEmit_ISD_CLMUL_rr(VT, RetVT, Op0, Op1);
14211 case ISD::CONCAT_VECTORS: return fastEmit_ISD_CONCAT_VECTORS_rr(VT, RetVT, Op0, Op1);
14212 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
14213 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
14214 case ISD::FMAXIMUM: return fastEmit_ISD_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
14215 case ISD::FMAXNUM: return fastEmit_ISD_FMAXNUM_rr(VT, RetVT, Op0, Op1);
14216 case ISD::FMAXNUM_IEEE: return fastEmit_ISD_FMAXNUM_IEEE_rr(VT, RetVT, Op0, Op1);
14217 case ISD::FMINIMUM: return fastEmit_ISD_FMINIMUM_rr(VT, RetVT, Op0, Op1);
14218 case ISD::FMINNUM: return fastEmit_ISD_FMINNUM_rr(VT, RetVT, Op0, Op1);
14219 case ISD::FMINNUM_IEEE: return fastEmit_ISD_FMINNUM_IEEE_rr(VT, RetVT, Op0, Op1);
14220 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
14221 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
14222 case ISD::GET_ACTIVE_LANE_MASK: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_rr(VT, RetVT, Op0, Op1);
14223 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
14224 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
14225 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
14226 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
14227 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
14228 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
14229 case ISD::SDIV: return fastEmit_ISD_SDIV_rr(VT, RetVT, Op0, Op1);
14230 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
14231 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
14232 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
14233 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
14234 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
14235 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
14236 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1);
14237 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1);
14238 case ISD::STRICT_FMAXIMUM: return fastEmit_ISD_STRICT_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
14239 case ISD::STRICT_FMAXNUM: return fastEmit_ISD_STRICT_FMAXNUM_rr(VT, RetVT, Op0, Op1);
14240 case ISD::STRICT_FMINIMUM: return fastEmit_ISD_STRICT_FMINIMUM_rr(VT, RetVT, Op0, Op1);
14241 case ISD::STRICT_FMINNUM: return fastEmit_ISD_STRICT_FMINNUM_rr(VT, RetVT, Op0, Op1);
14242 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1);
14243 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1);
14244 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
14245 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
14246 case ISD::UDIV: return fastEmit_ISD_UDIV_rr(VT, RetVT, Op0, Op1);
14247 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
14248 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
14249 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
14250 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
14251 default: return Register();
14252 }
14253}
14254
14255// FastEmit functions for AArch64ISD::DUPLANE64.
14256
14257Register fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14258 if (RetVT.SimpleTy != MVT::v2i64)
14259 return Register();
14260 if ((Subtarget->isNeonAvailable())) {
14261 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i64lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14262 }
14263 return Register();
14264}
14265
14266Register fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14267 if (RetVT.SimpleTy != MVT::v2f64)
14268 return Register();
14269 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i64lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14270}
14271
14272Register fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14273 switch (VT.SimpleTy) {
14274 case MVT::v2i64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14275 case MVT::v2f64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14276 default: return Register();
14277 }
14278}
14279
14280// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14281
14282Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14283 if (RetVT.SimpleTy != MVT::i64)
14284 return Register();
14285 if ((Subtarget->isNeonAvailable())) {
14286 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi64, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14287 }
14288 return Register();
14289}
14290
14291Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14292 if (RetVT.SimpleTy != MVT::f64)
14293 return Register();
14294 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi64, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14295}
14296
14297Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14298 switch (VT.SimpleTy) {
14299 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14300 case MVT::v2f64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14301 default: return Register();
14302 }
14303}
14304
14305// Top-level FastEmit function.
14306
14307Register fastEmit_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14308 switch (Opcode) {
14309 case AArch64ISD::DUPLANE64: return fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1);
14310 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1);
14311 default: return Register();
14312 }
14313}
14314
14315// FastEmit functions for AArch64ISD::DUPLANE32.
14316
14317Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14318 if ((Subtarget->isNeonAvailable())) {
14319 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i32lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14320 }
14321 return Register();
14322}
14323
14324Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14325 if ((Subtarget->isNeonAvailable())) {
14326 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i32lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14327 }
14328 return Register();
14329}
14330
14331Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14332switch (RetVT.SimpleTy) {
14333 case MVT::v2i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Op0, imm1);
14334 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Op0, imm1);
14335 default: return Register();
14336}
14337}
14338
14339Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14340 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i32lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14341}
14342
14343Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14344 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i32lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14345}
14346
14347Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14348switch (RetVT.SimpleTy) {
14349 case MVT::v2f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Op0, imm1);
14350 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Op0, imm1);
14351 default: return Register();
14352}
14353}
14354
14355Register fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14356 switch (VT.SimpleTy) {
14357 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14358 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14359 default: return Register();
14360 }
14361}
14362
14363// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14364
14365Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14366 if (RetVT.SimpleTy != MVT::i32)
14367 return Register();
14368 if ((Subtarget->isNeonAvailable())) {
14369 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi32, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14370 }
14371 return Register();
14372}
14373
14374Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14375 if (RetVT.SimpleTy != MVT::f32)
14376 return Register();
14377 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi32, RC: &AArch64::FPR32RegClass, Op0, Imm: imm1);
14378}
14379
14380Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14381 switch (VT.SimpleTy) {
14382 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14383 case MVT::v4f32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14384 default: return Register();
14385 }
14386}
14387
14388// Top-level FastEmit function.
14389
14390Register fastEmit_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14391 switch (Opcode) {
14392 case AArch64ISD::DUPLANE32: return fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1);
14393 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1);
14394 default: return Register();
14395 }
14396}
14397
14398// FastEmit functions for AArch64ISD::DUPLANE16.
14399
14400Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14401 if ((Subtarget->isNeonAvailable())) {
14402 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14403 }
14404 return Register();
14405}
14406
14407Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14408 if ((Subtarget->isNeonAvailable())) {
14409 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14410 }
14411 return Register();
14412}
14413
14414Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14415switch (RetVT.SimpleTy) {
14416 case MVT::v4i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Op0, imm1);
14417 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Op0, imm1);
14418 default: return Register();
14419}
14420}
14421
14422Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14423 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14424}
14425
14426Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14427 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14428}
14429
14430Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14431switch (RetVT.SimpleTy) {
14432 case MVT::v4f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Op0, imm1);
14433 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Op0, imm1);
14434 default: return Register();
14435}
14436}
14437
14438Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14439 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14440}
14441
14442Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14443 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14444}
14445
14446Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14447switch (RetVT.SimpleTy) {
14448 case MVT::v4bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(Op0, imm1);
14449 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(Op0, imm1);
14450 default: return Register();
14451}
14452}
14453
14454Register fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14455 switch (VT.SimpleTy) {
14456 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14457 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14458 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14459 default: return Register();
14460 }
14461}
14462
14463// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14464
14465Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14466 if (RetVT.SimpleTy != MVT::i32)
14467 return Register();
14468 if ((Subtarget->isNeonAvailable())) {
14469 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi16, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14470 }
14471 return Register();
14472}
14473
14474Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14475 if (RetVT.SimpleTy != MVT::f16)
14476 return Register();
14477 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi16, RC: &AArch64::FPR16RegClass, Op0, Imm: imm1);
14478}
14479
14480Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14481 if (RetVT.SimpleTy != MVT::bf16)
14482 return Register();
14483 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi16, RC: &AArch64::FPR16RegClass, Op0, Imm: imm1);
14484}
14485
14486Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14487 switch (VT.SimpleTy) {
14488 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14489 case MVT::v8f16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14490 case MVT::v8bf16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14491 default: return Register();
14492 }
14493}
14494
14495// Top-level FastEmit function.
14496
14497Register fastEmit_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14498 switch (Opcode) {
14499 case AArch64ISD::DUPLANE16: return fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1);
14500 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1);
14501 default: return Register();
14502 }
14503}
14504
14505// FastEmit functions for AArch64ISD::DUPLANE8.
14506
14507Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Register Op0, uint64_t imm1) {
14508 if ((Subtarget->isNeonAvailable())) {
14509 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i8lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14510 }
14511 return Register();
14512}
14513
14514Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Register Op0, uint64_t imm1) {
14515 if ((Subtarget->isNeonAvailable())) {
14516 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv16i8lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14517 }
14518 return Register();
14519}
14520
14521Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, Register Op0, uint64_t imm1) {
14522switch (RetVT.SimpleTy) {
14523 case MVT::v8i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Op0, imm1);
14524 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Op0, imm1);
14525 default: return Register();
14526}
14527}
14528
14529Register fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14530 switch (VT.SimpleTy) {
14531 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1);
14532 default: return Register();
14533 }
14534}
14535
14536// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14537
14538Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, Register Op0, uint64_t imm1) {
14539 if (RetVT.SimpleTy != MVT::i32)
14540 return Register();
14541 if ((Subtarget->isNeonAvailable())) {
14542 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi8, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14543 }
14544 return Register();
14545}
14546
14547Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14548 switch (VT.SimpleTy) {
14549 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1);
14550 default: return Register();
14551 }
14552}
14553
14554// Top-level FastEmit function.
14555
14556Register fastEmit_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14557 switch (Opcode) {
14558 case AArch64ISD::DUPLANE8: return fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1);
14559 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1);
14560 default: return Register();
14561 }
14562}
14563
14564// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14565
14566Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14567 if (RetVT.SimpleTy != MVT::i32)
14568 return Register();
14569 if ((Subtarget->hasNEON())) {
14570 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi8_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14571 }
14572 return Register();
14573}
14574
14575Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14576 if (RetVT.SimpleTy != MVT::i32)
14577 return Register();
14578 if ((Subtarget->hasNEON())) {
14579 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi16_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14580 }
14581 return Register();
14582}
14583
14584Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14585 if (RetVT.SimpleTy != MVT::i32)
14586 return Register();
14587 if ((Subtarget->hasNEON())) {
14588 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi32_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14589 }
14590 return Register();
14591}
14592
14593Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14594 if (RetVT.SimpleTy != MVT::i64)
14595 return Register();
14596 if ((Subtarget->hasNEON())) {
14597 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi64_idx0, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14598 }
14599 return Register();
14600}
14601
14602Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14603 switch (VT.SimpleTy) {
14604 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14605 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14606 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14607 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14608 default: return Register();
14609 }
14610}
14611
14612// Top-level FastEmit function.
14613
14614Register fastEmit_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14615 switch (Opcode) {
14616 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(VT, RetVT, Op0, imm1);
14617 default: return Register();
14618 }
14619}
14620
14621// FastEmit functions for ISD::SMAX.
14622
14623Register fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14624 if (RetVT.SimpleTy != MVT::i32)
14625 return Register();
14626 if ((Subtarget->hasCSSC())) {
14627 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMAXWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14628 }
14629 return Register();
14630}
14631
14632Register fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14633 switch (VT.SimpleTy) {
14634 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1);
14635 default: return Register();
14636 }
14637}
14638
14639// FastEmit functions for ISD::SMIN.
14640
14641Register fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14642 if (RetVT.SimpleTy != MVT::i32)
14643 return Register();
14644 if ((Subtarget->hasCSSC())) {
14645 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMINWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14646 }
14647 return Register();
14648}
14649
14650Register fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14651 switch (VT.SimpleTy) {
14652 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1);
14653 default: return Register();
14654 }
14655}
14656
14657// Top-level FastEmit function.
14658
14659Register fastEmit_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14660 switch (Opcode) {
14661 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1);
14662 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1);
14663 default: return Register();
14664 }
14665}
14666
14667// FastEmit functions for ISD::SMAX.
14668
14669Register fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14670 if (RetVT.SimpleTy != MVT::i64)
14671 return Register();
14672 if ((Subtarget->hasCSSC())) {
14673 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMAXXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14674 }
14675 return Register();
14676}
14677
14678Register fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14679 switch (VT.SimpleTy) {
14680 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1);
14681 default: return Register();
14682 }
14683}
14684
14685// FastEmit functions for ISD::SMIN.
14686
14687Register fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14688 if (RetVT.SimpleTy != MVT::i64)
14689 return Register();
14690 if ((Subtarget->hasCSSC())) {
14691 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMINXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14692 }
14693 return Register();
14694}
14695
14696Register fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14697 switch (VT.SimpleTy) {
14698 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1);
14699 default: return Register();
14700 }
14701}
14702
14703// Top-level FastEmit function.
14704
14705Register fastEmit_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14706 switch (Opcode) {
14707 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1);
14708 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1);
14709 default: return Register();
14710 }
14711}
14712
14713// FastEmit functions for ISD::UMAX.
14714
14715Register fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14716 if (RetVT.SimpleTy != MVT::i32)
14717 return Register();
14718 if ((Subtarget->hasCSSC())) {
14719 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMAXWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14720 }
14721 return Register();
14722}
14723
14724Register fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14725 switch (VT.SimpleTy) {
14726 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1);
14727 default: return Register();
14728 }
14729}
14730
14731// FastEmit functions for ISD::UMIN.
14732
14733Register fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14734 if (RetVT.SimpleTy != MVT::i32)
14735 return Register();
14736 if ((Subtarget->hasCSSC())) {
14737 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMINWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14738 }
14739 return Register();
14740}
14741
14742Register fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14743 switch (VT.SimpleTy) {
14744 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1);
14745 default: return Register();
14746 }
14747}
14748
14749// Top-level FastEmit function.
14750
14751Register fastEmit_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14752 switch (Opcode) {
14753 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1);
14754 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1);
14755 default: return Register();
14756 }
14757}
14758
14759// FastEmit functions for ISD::UMAX.
14760
14761Register fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14762 if (RetVT.SimpleTy != MVT::i64)
14763 return Register();
14764 if ((Subtarget->hasCSSC())) {
14765 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMAXXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14766 }
14767 return Register();
14768}
14769
14770Register fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14771 switch (VT.SimpleTy) {
14772 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1);
14773 default: return Register();
14774 }
14775}
14776
14777// FastEmit functions for ISD::UMIN.
14778
14779Register fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14780 if (RetVT.SimpleTy != MVT::i64)
14781 return Register();
14782 if ((Subtarget->hasCSSC())) {
14783 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMINXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14784 }
14785 return Register();
14786}
14787
14788Register fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14789 switch (VT.SimpleTy) {
14790 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1);
14791 default: return Register();
14792 }
14793}
14794
14795// Top-level FastEmit function.
14796
14797Register fastEmit_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14798 switch (Opcode) {
14799 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1);
14800 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1);
14801 default: return Register();
14802 }
14803}
14804
14805// FastEmit functions for ISD::Constant.
14806
14807Register fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
14808 if (RetVT.SimpleTy != MVT::i32)
14809 return Register();
14810 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVi32imm, RC: &AArch64::GPR32RegClass, Imm: imm0);
14811}
14812
14813Register fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) {
14814 if (RetVT.SimpleTy != MVT::i64)
14815 return Register();
14816 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVi64imm, RC: &AArch64::GPR64RegClass, Imm: imm0);
14817}
14818
14819Register fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
14820 switch (VT.SimpleTy) {
14821 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
14822 case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0);
14823 default: return Register();
14824 }
14825}
14826
14827// Top-level FastEmit function.
14828
14829Register fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
14830 if (VT == MVT::i32 && Predicate_imm0_255(Imm: imm0))
14831 if (Register Reg = fastEmit_i_Predicate_imm0_255(VT, RetVT, Opcode, imm0))
14832 return Reg;
14833
14834 if (VT == MVT::i32 && Predicate_simm6_32b(Imm: imm0))
14835 if (Register Reg = fastEmit_i_Predicate_simm6_32b(VT, RetVT, Opcode, imm0))
14836 return Reg;
14837
14838 switch (Opcode) {
14839 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
14840 default: return Register();
14841 }
14842}
14843
14844// FastEmit functions for AArch64ISD::FMOV.
14845
14846Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(uint64_t imm0) {
14847 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
14848 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv4f16_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
14849 }
14850 return Register();
14851}
14852
14853Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(uint64_t imm0) {
14854 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
14855 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv8f16_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14856 }
14857 return Register();
14858}
14859
14860Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(uint64_t imm0) {
14861 if ((Subtarget->isNeonAvailable())) {
14862 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv2f32_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
14863 }
14864 return Register();
14865}
14866
14867Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(uint64_t imm0) {
14868 if ((Subtarget->isNeonAvailable())) {
14869 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv4f32_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14870 }
14871 return Register();
14872}
14873
14874Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(uint64_t imm0) {
14875 if ((Subtarget->isNeonAvailable())) {
14876 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv2f64_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14877 }
14878 return Register();
14879}
14880
14881Register fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
14882switch (RetVT.SimpleTy) {
14883 case MVT::v4f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(imm0);
14884 case MVT::v8f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(imm0);
14885 case MVT::v2f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(imm0);
14886 case MVT::v4f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(imm0);
14887 case MVT::v2f64: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(imm0);
14888 default: return Register();
14889}
14890}
14891
14892Register fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
14893 switch (VT.SimpleTy) {
14894 case MVT::i32: return fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
14895 default: return Register();
14896 }
14897}
14898
14899// FastEmit functions for AArch64ISD::MOVI.
14900
14901Register fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(uint64_t imm0) {
14902 if ((Subtarget->isNeonAvailable())) {
14903 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv8b_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
14904 }
14905 return Register();
14906}
14907
14908Register fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(uint64_t imm0) {
14909 if ((Subtarget->isNeonAvailable())) {
14910 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv16b_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14911 }
14912 return Register();
14913}
14914
14915Register fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
14916switch (RetVT.SimpleTy) {
14917 case MVT::v8i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(imm0);
14918 case MVT::v16i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(imm0);
14919 default: return Register();
14920}
14921}
14922
14923Register fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
14924 switch (VT.SimpleTy) {
14925 case MVT::i32: return fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
14926 default: return Register();
14927 }
14928}
14929
14930// FastEmit functions for AArch64ISD::MOVIedit.
14931
14932Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(uint64_t imm0) {
14933 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVID, RC: &AArch64::FPR64RegClass, Imm: imm0);
14934}
14935
14936Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(uint64_t imm0) {
14937 if ((Subtarget->isNeonAvailable())) {
14938 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv2d_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14939 }
14940 return Register();
14941}
14942
14943Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
14944switch (RetVT.SimpleTy) {
14945 case MVT::f64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(imm0);
14946 case MVT::v2i64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(imm0);
14947 default: return Register();
14948}
14949}
14950
14951Register fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
14952 switch (VT.SimpleTy) {
14953 case MVT::i32: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
14954 default: return Register();
14955 }
14956}
14957
14958// Top-level FastEmit function.
14959
14960Register fastEmit_i_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
14961 switch (Opcode) {
14962 case AArch64ISD::FMOV: return fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(VT, RetVT, imm0);
14963 case AArch64ISD::MOVI: return fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(VT, RetVT, imm0);
14964 case AArch64ISD::MOVIedit: return fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(VT, RetVT, imm0);
14965 default: return Register();
14966 }
14967}
14968
14969// FastEmit functions for AArch64ISD::RDSVL.
14970
14971Register fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(MVT RetVT, uint64_t imm0) {
14972 if (RetVT.SimpleTy != MVT::i64)
14973 return Register();
14974 if ((Subtarget->hasSME())) {
14975 return fastEmitInst_i(MachineInstOpcode: AArch64::RDSVLI_XI, RC: &AArch64::GPR64RegClass, Imm: imm0);
14976 }
14977 return Register();
14978}
14979
14980Register fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(MVT VT, MVT RetVT, uint64_t imm0) {
14981 switch (VT.SimpleTy) {
14982 case MVT::i32: return fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(RetVT, imm0);
14983 default: return Register();
14984 }
14985}
14986
14987// Top-level FastEmit function.
14988
14989Register fastEmit_i_Predicate_simm6_32b(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
14990 switch (Opcode) {
14991 case AArch64ISD::RDSVL: return fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(VT, RetVT, imm0);
14992 default: return Register();
14993 }
14994}
14995
14996