1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the AArch64 target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_imm0_31(int64_t Imm) {
12
13 return ((uint64_t)Imm) < 32;
14
15}
16static bool Predicate_imm0_63(int64_t Imm) {
17
18 return ((uint64_t)Imm) < 64;
19
20}
21static bool Predicate_imm32_0_31(int64_t Imm) {
22
23 return ((uint64_t)Imm) < 32;
24
25}
26static bool Predicate_tbz_imm0_31_diag(int64_t Imm) {
27
28 return (((uint32_t)Imm) < 32);
29
30}
31static bool Predicate_tbz_imm32_63(int64_t Imm) {
32
33 return (((uint32_t)Imm) > 31) && (((uint32_t)Imm) < 64);
34
35}
36static bool Predicate_VectorIndexD(int64_t Imm) {
37 return ((uint64_t)Imm) < 2;
38}
39static bool Predicate_VectorIndexS(int64_t Imm) {
40 return ((uint64_t)Imm) < 4;
41}
42static bool Predicate_VectorIndexH(int64_t Imm) {
43 return ((uint64_t)Imm) < 8;
44}
45static bool Predicate_VectorIndexB(int64_t Imm) {
46 return ((uint64_t)Imm) < 16;
47}
48static bool Predicate_VectorIndex0(int64_t Imm) {
49 return ((uint64_t)Imm) == 0;
50}
51static bool Predicate_imm0_255(int64_t Imm) {
52
53 return ((uint32_t)Imm) < 256;
54
55}
56static bool Predicate_simm8_32b(int64_t Imm) {
57 return Imm >= -128 && Imm < 128;
58}
59static bool Predicate_simm8_64b(int64_t Imm) {
60 return Imm >= -128 && Imm < 128;
61}
62static bool Predicate_uimm8_32b(int64_t Imm) {
63 return Imm >= 0 && Imm < 256;
64}
65static bool Predicate_uimm8_64b(int64_t Imm) {
66 return Imm >= 0 && Imm < 256;
67}
68static bool Predicate_simm6_32b(int64_t Imm) {
69 return Imm >= -32 && Imm < 32;
70}
71
72
73// FastEmit functions for AArch64ISD::ENTRY_PSTATE_SM.
74
75Register fastEmit_AArch64ISD_ENTRY_PSTATE_SM_MVT_i64_(MVT RetVT) {
76 if (RetVT.SimpleTy != MVT::i64)
77 return Register();
78 return fastEmitInst_(MachineInstOpcode: AArch64::EntryPStateSM, RC: &AArch64::GPR64RegClass);
79}
80
81Register fastEmit_AArch64ISD_ENTRY_PSTATE_SM_(MVT VT, MVT RetVT) {
82 switch (VT.SimpleTy) {
83 case MVT::i64: return fastEmit_AArch64ISD_ENTRY_PSTATE_SM_MVT_i64_(RetVT);
84 default: return Register();
85 }
86}
87
88// FastEmit functions for AArch64ISD::THREAD_POINTER.
89
90Register fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(MVT RetVT) {
91 if (RetVT.SimpleTy != MVT::i64)
92 return Register();
93 return fastEmitInst_(MachineInstOpcode: AArch64::MOVbaseTLS, RC: &AArch64::GPR64RegClass);
94}
95
96Register fastEmit_AArch64ISD_THREAD_POINTER_(MVT VT, MVT RetVT) {
97 switch (VT.SimpleTy) {
98 case MVT::i64: return fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(RetVT);
99 default: return Register();
100 }
101}
102
103// Top-level FastEmit function.
104
105Register fastEmit_(MVT VT, MVT RetVT, unsigned Opcode) override {
106 switch (Opcode) {
107 case AArch64ISD::ENTRY_PSTATE_SM: return fastEmit_AArch64ISD_ENTRY_PSTATE_SM_(VT, RetVT);
108 case AArch64ISD::THREAD_POINTER: return fastEmit_AArch64ISD_THREAD_POINTER_(VT, RetVT);
109 default: return Register();
110 }
111}
112
113// FastEmit functions for AArch64ISD::CALL.
114
115Register fastEmit_AArch64ISD_CALL_MVT_i64_r(MVT RetVT, Register Op0) {
116 if (RetVT.SimpleTy != MVT::isVoid)
117 return Register();
118 if (( MF->getSubtarget<AArch64Subtarget>().hardenSlsBlr() )) {
119 return fastEmitInst_r(MachineInstOpcode: AArch64::BLRNoIP, RC: &AArch64::GPR64noipRegClass, Op0);
120 }
121 if (( !MF->getSubtarget<AArch64Subtarget>().hardenSlsBlr() )) {
122 return fastEmitInst_r(MachineInstOpcode: AArch64::BLR, RC: &AArch64::GPR64RegClass, Op0);
123 }
124 return Register();
125}
126
127Register fastEmit_AArch64ISD_CALL_r(MVT VT, MVT RetVT, Register Op0) {
128 switch (VT.SimpleTy) {
129 case MVT::i64: return fastEmit_AArch64ISD_CALL_MVT_i64_r(RetVT, Op0);
130 default: return Register();
131 }
132}
133
134// FastEmit functions for AArch64ISD::COALESCER_BARRIER.
135
136Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_bf16_r(MVT RetVT, Register Op0) {
137 if (RetVT.SimpleTy != MVT::bf16)
138 return Register();
139 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR16, RC: &AArch64::FPR16RegClass, Op0);
140}
141
142Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f16_r(MVT RetVT, Register Op0) {
143 if (RetVT.SimpleTy != MVT::f16)
144 return Register();
145 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR16, RC: &AArch64::FPR16RegClass, Op0);
146}
147
148Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f32_r(MVT RetVT, Register Op0) {
149 if (RetVT.SimpleTy != MVT::f32)
150 return Register();
151 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR32, RC: &AArch64::FPR32RegClass, Op0);
152}
153
154Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f64_r(MVT RetVT, Register Op0) {
155 if (RetVT.SimpleTy != MVT::f64)
156 return Register();
157 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
158}
159
160Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f128_r(MVT RetVT, Register Op0) {
161 if (RetVT.SimpleTy != MVT::f128)
162 return Register();
163 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
164}
165
166Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i8_r(MVT RetVT, Register Op0) {
167 if (RetVT.SimpleTy != MVT::v8i8)
168 return Register();
169 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
170}
171
172Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v16i8_r(MVT RetVT, Register Op0) {
173 if (RetVT.SimpleTy != MVT::v16i8)
174 return Register();
175 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
176}
177
178Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i16_r(MVT RetVT, Register Op0) {
179 if (RetVT.SimpleTy != MVT::v4i16)
180 return Register();
181 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
182}
183
184Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i16_r(MVT RetVT, Register Op0) {
185 if (RetVT.SimpleTy != MVT::v8i16)
186 return Register();
187 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
188}
189
190Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i32_r(MVT RetVT, Register Op0) {
191 if (RetVT.SimpleTy != MVT::v2i32)
192 return Register();
193 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
194}
195
196Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i32_r(MVT RetVT, Register Op0) {
197 if (RetVT.SimpleTy != MVT::v4i32)
198 return Register();
199 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
200}
201
202Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1i64_r(MVT RetVT, Register Op0) {
203 if (RetVT.SimpleTy != MVT::v1i64)
204 return Register();
205 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
206}
207
208Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i64_r(MVT RetVT, Register Op0) {
209 if (RetVT.SimpleTy != MVT::v2i64)
210 return Register();
211 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
212}
213
214Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f16_r(MVT RetVT, Register Op0) {
215 if (RetVT.SimpleTy != MVT::v4f16)
216 return Register();
217 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
218}
219
220Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8f16_r(MVT RetVT, Register Op0) {
221 if (RetVT.SimpleTy != MVT::v8f16)
222 return Register();
223 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
224}
225
226Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4bf16_r(MVT RetVT, Register Op0) {
227 if (RetVT.SimpleTy != MVT::v4bf16)
228 return Register();
229 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
230}
231
232Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8bf16_r(MVT RetVT, Register Op0) {
233 if (RetVT.SimpleTy != MVT::v8bf16)
234 return Register();
235 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
236}
237
238Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f32_r(MVT RetVT, Register Op0) {
239 if (RetVT.SimpleTy != MVT::v2f32)
240 return Register();
241 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
242}
243
244Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f32_r(MVT RetVT, Register Op0) {
245 if (RetVT.SimpleTy != MVT::v4f32)
246 return Register();
247 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
248}
249
250Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1f64_r(MVT RetVT, Register Op0) {
251 if (RetVT.SimpleTy != MVT::v1f64)
252 return Register();
253 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
254}
255
256Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f64_r(MVT RetVT, Register Op0) {
257 if (RetVT.SimpleTy != MVT::v2f64)
258 return Register();
259 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
260}
261
262Register fastEmit_AArch64ISD_COALESCER_BARRIER_r(MVT VT, MVT RetVT, Register Op0) {
263 switch (VT.SimpleTy) {
264 case MVT::bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_bf16_r(RetVT, Op0);
265 case MVT::f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f16_r(RetVT, Op0);
266 case MVT::f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f32_r(RetVT, Op0);
267 case MVT::f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f64_r(RetVT, Op0);
268 case MVT::f128: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f128_r(RetVT, Op0);
269 case MVT::v8i8: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i8_r(RetVT, Op0);
270 case MVT::v16i8: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v16i8_r(RetVT, Op0);
271 case MVT::v4i16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i16_r(RetVT, Op0);
272 case MVT::v8i16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i16_r(RetVT, Op0);
273 case MVT::v2i32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i32_r(RetVT, Op0);
274 case MVT::v4i32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i32_r(RetVT, Op0);
275 case MVT::v1i64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1i64_r(RetVT, Op0);
276 case MVT::v2i64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i64_r(RetVT, Op0);
277 case MVT::v4f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f16_r(RetVT, Op0);
278 case MVT::v8f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8f16_r(RetVT, Op0);
279 case MVT::v4bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4bf16_r(RetVT, Op0);
280 case MVT::v8bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8bf16_r(RetVT, Op0);
281 case MVT::v2f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f32_r(RetVT, Op0);
282 case MVT::v4f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f32_r(RetVT, Op0);
283 case MVT::v1f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1f64_r(RetVT, Op0);
284 case MVT::v2f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f64_r(RetVT, Op0);
285 default: return Register();
286 }
287}
288
289// FastEmit functions for AArch64ISD::DUP.
290
291Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Register Op0) {
292 if ((Subtarget->isNeonAvailable())) {
293 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv8i8gpr, RC: &AArch64::FPR64RegClass, Op0);
294 }
295 return Register();
296}
297
298Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Register Op0) {
299 if ((Subtarget->isNeonAvailable())) {
300 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv16i8gpr, RC: &AArch64::FPR128RegClass, Op0);
301 }
302 return Register();
303}
304
305Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Register Op0) {
306 if ((Subtarget->isNeonAvailable())) {
307 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv4i16gpr, RC: &AArch64::FPR64RegClass, Op0);
308 }
309 return Register();
310}
311
312Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Register Op0) {
313 if ((Subtarget->isNeonAvailable())) {
314 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv8i16gpr, RC: &AArch64::FPR128RegClass, Op0);
315 }
316 return Register();
317}
318
319Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Register Op0) {
320 if ((Subtarget->isNeonAvailable())) {
321 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv2i32gpr, RC: &AArch64::FPR64RegClass, Op0);
322 }
323 return Register();
324}
325
326Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Register Op0) {
327 if ((Subtarget->isNeonAvailable())) {
328 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv4i32gpr, RC: &AArch64::FPR128RegClass, Op0);
329 }
330 return Register();
331}
332
333Register fastEmit_AArch64ISD_DUP_MVT_i32_r(MVT RetVT, Register Op0) {
334switch (RetVT.SimpleTy) {
335 case MVT::v8i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Op0);
336 case MVT::v16i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Op0);
337 case MVT::v4i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Op0);
338 case MVT::v8i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Op0);
339 case MVT::v2i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Op0);
340 case MVT::v4i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Op0);
341 default: return Register();
342}
343}
344
345Register fastEmit_AArch64ISD_DUP_MVT_i64_r(MVT RetVT, Register Op0) {
346 if (RetVT.SimpleTy != MVT::v2i64)
347 return Register();
348 if ((Subtarget->isNeonAvailable())) {
349 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv2i64gpr, RC: &AArch64::FPR128RegClass, Op0);
350 }
351 return Register();
352}
353
354Register fastEmit_AArch64ISD_DUP_r(MVT VT, MVT RetVT, Register Op0) {
355 switch (VT.SimpleTy) {
356 case MVT::i32: return fastEmit_AArch64ISD_DUP_MVT_i32_r(RetVT, Op0);
357 case MVT::i64: return fastEmit_AArch64ISD_DUP_MVT_i64_r(RetVT, Op0);
358 default: return Register();
359 }
360}
361
362// FastEmit functions for AArch64ISD::FCVTXN.
363
364Register fastEmit_AArch64ISD_FCVTXN_MVT_f64_r(MVT RetVT, Register Op0) {
365 if (RetVT.SimpleTy != MVT::f32)
366 return Register();
367 if ((Subtarget->isNeonAvailable())) {
368 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTXNv1i64, RC: &AArch64::FPR32RegClass, Op0);
369 }
370 return Register();
371}
372
373Register fastEmit_AArch64ISD_FCVTXN_MVT_v2f64_r(MVT RetVT, Register Op0) {
374 if (RetVT.SimpleTy != MVT::v2f32)
375 return Register();
376 if ((Subtarget->isNeonAvailable())) {
377 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTXNv2f32, RC: &AArch64::FPR64RegClass, Op0);
378 }
379 return Register();
380}
381
382Register fastEmit_AArch64ISD_FCVTXN_r(MVT VT, MVT RetVT, Register Op0) {
383 switch (VT.SimpleTy) {
384 case MVT::f64: return fastEmit_AArch64ISD_FCVTXN_MVT_f64_r(RetVT, Op0);
385 case MVT::v2f64: return fastEmit_AArch64ISD_FCVTXN_MVT_v2f64_r(RetVT, Op0);
386 default: return Register();
387 }
388}
389
390// FastEmit functions for AArch64ISD::FRECPE.
391
392Register fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(MVT RetVT, Register Op0) {
393 if (RetVT.SimpleTy != MVT::v2f32)
394 return Register();
395 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv2f32, RC: &AArch64::FPR64RegClass, Op0);
396}
397
398Register fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(MVT RetVT, Register Op0) {
399 if (RetVT.SimpleTy != MVT::v4f32)
400 return Register();
401 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv4f32, RC: &AArch64::FPR128RegClass, Op0);
402}
403
404Register fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(MVT RetVT, Register Op0) {
405 if (RetVT.SimpleTy != MVT::v2f64)
406 return Register();
407 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv2f64, RC: &AArch64::FPR128RegClass, Op0);
408}
409
410Register fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
411 if (RetVT.SimpleTy != MVT::nxv8f16)
412 return Register();
413 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
414 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
415 }
416 return Register();
417}
418
419Register fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
420 if (RetVT.SimpleTy != MVT::nxv4f32)
421 return Register();
422 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
423 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
424 }
425 return Register();
426}
427
428Register fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
429 if (RetVT.SimpleTy != MVT::nxv2f64)
430 return Register();
431 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
432 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
433 }
434 return Register();
435}
436
437Register fastEmit_AArch64ISD_FRECPE_r(MVT VT, MVT RetVT, Register Op0) {
438 switch (VT.SimpleTy) {
439 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(RetVT, Op0);
440 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(RetVT, Op0);
441 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(RetVT, Op0);
442 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(RetVT, Op0);
443 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(RetVT, Op0);
444 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(RetVT, Op0);
445 default: return Register();
446 }
447}
448
449// FastEmit functions for AArch64ISD::FRSQRTE.
450
451Register fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(MVT RetVT, Register Op0) {
452 if (RetVT.SimpleTy != MVT::v2f32)
453 return Register();
454 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv2f32, RC: &AArch64::FPR64RegClass, Op0);
455}
456
457Register fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(MVT RetVT, Register Op0) {
458 if (RetVT.SimpleTy != MVT::v4f32)
459 return Register();
460 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv4f32, RC: &AArch64::FPR128RegClass, Op0);
461}
462
463Register fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(MVT RetVT, Register Op0) {
464 if (RetVT.SimpleTy != MVT::v2f64)
465 return Register();
466 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv2f64, RC: &AArch64::FPR128RegClass, Op0);
467}
468
469Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
470 if (RetVT.SimpleTy != MVT::nxv8f16)
471 return Register();
472 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
473 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
474 }
475 return Register();
476}
477
478Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
479 if (RetVT.SimpleTy != MVT::nxv4f32)
480 return Register();
481 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
482 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
483 }
484 return Register();
485}
486
487Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
488 if (RetVT.SimpleTy != MVT::nxv2f64)
489 return Register();
490 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
491 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
492 }
493 return Register();
494}
495
496Register fastEmit_AArch64ISD_FRSQRTE_r(MVT VT, MVT RetVT, Register Op0) {
497 switch (VT.SimpleTy) {
498 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(RetVT, Op0);
499 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(RetVT, Op0);
500 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(RetVT, Op0);
501 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(RetVT, Op0);
502 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(RetVT, Op0);
503 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(RetVT, Op0);
504 default: return Register();
505 }
506}
507
508// FastEmit functions for AArch64ISD::PROBED_ALLOCA.
509
510Register fastEmit_AArch64ISD_PROBED_ALLOCA_MVT_i64_r(MVT RetVT, Register Op0) {
511 if (RetVT.SimpleTy != MVT::isVoid)
512 return Register();
513 return fastEmitInst_r(MachineInstOpcode: AArch64::PROBED_STACKALLOC_DYN, RC: &AArch64::GPR64commonRegClass, Op0);
514}
515
516Register fastEmit_AArch64ISD_PROBED_ALLOCA_r(MVT VT, MVT RetVT, Register Op0) {
517 switch (VT.SimpleTy) {
518 case MVT::i64: return fastEmit_AArch64ISD_PROBED_ALLOCA_MVT_i64_r(RetVT, Op0);
519 default: return Register();
520 }
521}
522
523// FastEmit functions for AArch64ISD::REV16.
524
525Register fastEmit_AArch64ISD_REV16_MVT_i32_r(MVT RetVT, Register Op0) {
526 if (RetVT.SimpleTy != MVT::i32)
527 return Register();
528 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16Wr, RC: &AArch64::GPR32RegClass, Op0);
529}
530
531Register fastEmit_AArch64ISD_REV16_MVT_i64_r(MVT RetVT, Register Op0) {
532 if (RetVT.SimpleTy != MVT::i64)
533 return Register();
534 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16Xr, RC: &AArch64::GPR64RegClass, Op0);
535}
536
537Register fastEmit_AArch64ISD_REV16_r(MVT VT, MVT RetVT, Register Op0) {
538 switch (VT.SimpleTy) {
539 case MVT::i32: return fastEmit_AArch64ISD_REV16_MVT_i32_r(RetVT, Op0);
540 case MVT::i64: return fastEmit_AArch64ISD_REV16_MVT_i64_r(RetVT, Op0);
541 default: return Register();
542 }
543}
544
545// FastEmit functions for AArch64ISD::REV32.
546
547Register fastEmit_AArch64ISD_REV32_MVT_v8i8_r(MVT RetVT, Register Op0) {
548 if (RetVT.SimpleTy != MVT::v8i8)
549 return Register();
550 if ((Subtarget->isNeonAvailable())) {
551 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
552 }
553 return Register();
554}
555
556Register fastEmit_AArch64ISD_REV32_MVT_v16i8_r(MVT RetVT, Register Op0) {
557 if (RetVT.SimpleTy != MVT::v16i8)
558 return Register();
559 if ((Subtarget->isNeonAvailable())) {
560 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
561 }
562 return Register();
563}
564
565Register fastEmit_AArch64ISD_REV32_MVT_v4i16_r(MVT RetVT, Register Op0) {
566 if (RetVT.SimpleTy != MVT::v4i16)
567 return Register();
568 if ((Subtarget->isNeonAvailable())) {
569 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
570 }
571 return Register();
572}
573
574Register fastEmit_AArch64ISD_REV32_MVT_v8i16_r(MVT RetVT, Register Op0) {
575 if (RetVT.SimpleTy != MVT::v8i16)
576 return Register();
577 if ((Subtarget->isNeonAvailable())) {
578 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
579 }
580 return Register();
581}
582
583Register fastEmit_AArch64ISD_REV32_MVT_v4f16_r(MVT RetVT, Register Op0) {
584 if (RetVT.SimpleTy != MVT::v4f16)
585 return Register();
586 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
587}
588
589Register fastEmit_AArch64ISD_REV32_MVT_v8f16_r(MVT RetVT, Register Op0) {
590 if (RetVT.SimpleTy != MVT::v8f16)
591 return Register();
592 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
593}
594
595Register fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(MVT RetVT, Register Op0) {
596 if (RetVT.SimpleTy != MVT::v4bf16)
597 return Register();
598 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
599}
600
601Register fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(MVT RetVT, Register Op0) {
602 if (RetVT.SimpleTy != MVT::v8bf16)
603 return Register();
604 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
605}
606
607Register fastEmit_AArch64ISD_REV32_r(MVT VT, MVT RetVT, Register Op0) {
608 switch (VT.SimpleTy) {
609 case MVT::v8i8: return fastEmit_AArch64ISD_REV32_MVT_v8i8_r(RetVT, Op0);
610 case MVT::v16i8: return fastEmit_AArch64ISD_REV32_MVT_v16i8_r(RetVT, Op0);
611 case MVT::v4i16: return fastEmit_AArch64ISD_REV32_MVT_v4i16_r(RetVT, Op0);
612 case MVT::v8i16: return fastEmit_AArch64ISD_REV32_MVT_v8i16_r(RetVT, Op0);
613 case MVT::v4f16: return fastEmit_AArch64ISD_REV32_MVT_v4f16_r(RetVT, Op0);
614 case MVT::v8f16: return fastEmit_AArch64ISD_REV32_MVT_v8f16_r(RetVT, Op0);
615 case MVT::v4bf16: return fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(RetVT, Op0);
616 case MVT::v8bf16: return fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(RetVT, Op0);
617 default: return Register();
618 }
619}
620
621// FastEmit functions for AArch64ISD::REV64.
622
623Register fastEmit_AArch64ISD_REV64_MVT_v8i8_r(MVT RetVT, Register Op0) {
624 if (RetVT.SimpleTy != MVT::v8i8)
625 return Register();
626 if ((Subtarget->isNeonAvailable())) {
627 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
628 }
629 return Register();
630}
631
632Register fastEmit_AArch64ISD_REV64_MVT_v16i8_r(MVT RetVT, Register Op0) {
633 if (RetVT.SimpleTy != MVT::v16i8)
634 return Register();
635 if ((Subtarget->isNeonAvailable())) {
636 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
637 }
638 return Register();
639}
640
641Register fastEmit_AArch64ISD_REV64_MVT_v4i16_r(MVT RetVT, Register Op0) {
642 if (RetVT.SimpleTy != MVT::v4i16)
643 return Register();
644 if ((Subtarget->isNeonAvailable())) {
645 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
646 }
647 return Register();
648}
649
650Register fastEmit_AArch64ISD_REV64_MVT_v8i16_r(MVT RetVT, Register Op0) {
651 if (RetVT.SimpleTy != MVT::v8i16)
652 return Register();
653 if ((Subtarget->isNeonAvailable())) {
654 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
655 }
656 return Register();
657}
658
659Register fastEmit_AArch64ISD_REV64_MVT_v2i32_r(MVT RetVT, Register Op0) {
660 if (RetVT.SimpleTy != MVT::v2i32)
661 return Register();
662 if ((Subtarget->isNeonAvailable())) {
663 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
664 }
665 return Register();
666}
667
668Register fastEmit_AArch64ISD_REV64_MVT_v4i32_r(MVT RetVT, Register Op0) {
669 if (RetVT.SimpleTy != MVT::v4i32)
670 return Register();
671 if ((Subtarget->isNeonAvailable())) {
672 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
673 }
674 return Register();
675}
676
677Register fastEmit_AArch64ISD_REV64_MVT_v4f16_r(MVT RetVT, Register Op0) {
678 if (RetVT.SimpleTy != MVT::v4f16)
679 return Register();
680 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
681}
682
683Register fastEmit_AArch64ISD_REV64_MVT_v8f16_r(MVT RetVT, Register Op0) {
684 if (RetVT.SimpleTy != MVT::v8f16)
685 return Register();
686 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
687}
688
689Register fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(MVT RetVT, Register Op0) {
690 if (RetVT.SimpleTy != MVT::v4bf16)
691 return Register();
692 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
693}
694
695Register fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(MVT RetVT, Register Op0) {
696 if (RetVT.SimpleTy != MVT::v8bf16)
697 return Register();
698 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
699}
700
701Register fastEmit_AArch64ISD_REV64_MVT_v2f32_r(MVT RetVT, Register Op0) {
702 if (RetVT.SimpleTy != MVT::v2f32)
703 return Register();
704 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
705}
706
707Register fastEmit_AArch64ISD_REV64_MVT_v4f32_r(MVT RetVT, Register Op0) {
708 if (RetVT.SimpleTy != MVT::v4f32)
709 return Register();
710 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
711}
712
713Register fastEmit_AArch64ISD_REV64_r(MVT VT, MVT RetVT, Register Op0) {
714 switch (VT.SimpleTy) {
715 case MVT::v8i8: return fastEmit_AArch64ISD_REV64_MVT_v8i8_r(RetVT, Op0);
716 case MVT::v16i8: return fastEmit_AArch64ISD_REV64_MVT_v16i8_r(RetVT, Op0);
717 case MVT::v4i16: return fastEmit_AArch64ISD_REV64_MVT_v4i16_r(RetVT, Op0);
718 case MVT::v8i16: return fastEmit_AArch64ISD_REV64_MVT_v8i16_r(RetVT, Op0);
719 case MVT::v2i32: return fastEmit_AArch64ISD_REV64_MVT_v2i32_r(RetVT, Op0);
720 case MVT::v4i32: return fastEmit_AArch64ISD_REV64_MVT_v4i32_r(RetVT, Op0);
721 case MVT::v4f16: return fastEmit_AArch64ISD_REV64_MVT_v4f16_r(RetVT, Op0);
722 case MVT::v8f16: return fastEmit_AArch64ISD_REV64_MVT_v8f16_r(RetVT, Op0);
723 case MVT::v4bf16: return fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(RetVT, Op0);
724 case MVT::v8bf16: return fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(RetVT, Op0);
725 case MVT::v2f32: return fastEmit_AArch64ISD_REV64_MVT_v2f32_r(RetVT, Op0);
726 case MVT::v4f32: return fastEmit_AArch64ISD_REV64_MVT_v4f32_r(RetVT, Op0);
727 default: return Register();
728 }
729}
730
731// FastEmit functions for AArch64ISD::SADDLP.
732
733Register fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(MVT RetVT, Register Op0) {
734 if (RetVT.SimpleTy != MVT::v4i16)
735 return Register();
736 if ((Subtarget->isNeonAvailable())) {
737 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv8i8_v4i16, RC: &AArch64::FPR64RegClass, Op0);
738 }
739 return Register();
740}
741
742Register fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(MVT RetVT, Register Op0) {
743 if (RetVT.SimpleTy != MVT::v8i16)
744 return Register();
745 if ((Subtarget->isNeonAvailable())) {
746 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv16i8_v8i16, RC: &AArch64::FPR128RegClass, Op0);
747 }
748 return Register();
749}
750
751Register fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(MVT RetVT, Register Op0) {
752 if (RetVT.SimpleTy != MVT::v2i32)
753 return Register();
754 if ((Subtarget->isNeonAvailable())) {
755 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv4i16_v2i32, RC: &AArch64::FPR64RegClass, Op0);
756 }
757 return Register();
758}
759
760Register fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(MVT RetVT, Register Op0) {
761 if (RetVT.SimpleTy != MVT::v4i32)
762 return Register();
763 if ((Subtarget->isNeonAvailable())) {
764 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv8i16_v4i32, RC: &AArch64::FPR128RegClass, Op0);
765 }
766 return Register();
767}
768
769Register fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(MVT RetVT, Register Op0) {
770 if (RetVT.SimpleTy != MVT::v1i64)
771 return Register();
772 if ((Subtarget->isNeonAvailable())) {
773 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv2i32_v1i64, RC: &AArch64::FPR64RegClass, Op0);
774 }
775 return Register();
776}
777
778Register fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(MVT RetVT, Register Op0) {
779 if (RetVT.SimpleTy != MVT::v2i64)
780 return Register();
781 if ((Subtarget->isNeonAvailable())) {
782 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv4i32_v2i64, RC: &AArch64::FPR128RegClass, Op0);
783 }
784 return Register();
785}
786
787Register fastEmit_AArch64ISD_SADDLP_r(MVT VT, MVT RetVT, Register Op0) {
788 switch (VT.SimpleTy) {
789 case MVT::v8i8: return fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(RetVT, Op0);
790 case MVT::v16i8: return fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(RetVT, Op0);
791 case MVT::v4i16: return fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(RetVT, Op0);
792 case MVT::v8i16: return fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(RetVT, Op0);
793 case MVT::v2i32: return fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(RetVT, Op0);
794 case MVT::v4i32: return fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(RetVT, Op0);
795 default: return Register();
796 }
797}
798
799// FastEmit functions for AArch64ISD::SITOF.
800
801Register fastEmit_AArch64ISD_SITOF_MVT_f16_r(MVT RetVT, Register Op0) {
802 if (RetVT.SimpleTy != MVT::f16)
803 return Register();
804 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
805 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i16, RC: &AArch64::FPR16RegClass, Op0);
806 }
807 return Register();
808}
809
810Register fastEmit_AArch64ISD_SITOF_MVT_f32_r(MVT RetVT, Register Op0) {
811 if (RetVT.SimpleTy != MVT::f32)
812 return Register();
813 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
814 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i32, RC: &AArch64::FPR32RegClass, Op0);
815 }
816 return Register();
817}
818
819Register fastEmit_AArch64ISD_SITOF_MVT_f64_r(MVT RetVT, Register Op0) {
820 if (RetVT.SimpleTy != MVT::f64)
821 return Register();
822 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
823 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i64, RC: &AArch64::FPR64RegClass, Op0);
824 }
825 return Register();
826}
827
828Register fastEmit_AArch64ISD_SITOF_r(MVT VT, MVT RetVT, Register Op0) {
829 switch (VT.SimpleTy) {
830 case MVT::f16: return fastEmit_AArch64ISD_SITOF_MVT_f16_r(RetVT, Op0);
831 case MVT::f32: return fastEmit_AArch64ISD_SITOF_MVT_f32_r(RetVT, Op0);
832 case MVT::f64: return fastEmit_AArch64ISD_SITOF_MVT_f64_r(RetVT, Op0);
833 default: return Register();
834 }
835}
836
837// FastEmit functions for AArch64ISD::SQABS.
838
839Register fastEmit_AArch64ISD_SQABS_MVT_f32_r(MVT RetVT, Register Op0) {
840 if (RetVT.SimpleTy != MVT::f32)
841 return Register();
842 if ((Subtarget->isNeonAvailable())) {
843 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv1i32, RC: &AArch64::FPR32RegClass, Op0);
844 }
845 return Register();
846}
847
848Register fastEmit_AArch64ISD_SQABS_MVT_f64_r(MVT RetVT, Register Op0) {
849 if (RetVT.SimpleTy != MVT::f64)
850 return Register();
851 if ((Subtarget->isNeonAvailable())) {
852 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
853 }
854 return Register();
855}
856
857Register fastEmit_AArch64ISD_SQABS_r(MVT VT, MVT RetVT, Register Op0) {
858 switch (VT.SimpleTy) {
859 case MVT::f32: return fastEmit_AArch64ISD_SQABS_MVT_f32_r(RetVT, Op0);
860 case MVT::f64: return fastEmit_AArch64ISD_SQABS_MVT_f64_r(RetVT, Op0);
861 default: return Register();
862 }
863}
864
865// FastEmit functions for AArch64ISD::SQNEG.
866
867Register fastEmit_AArch64ISD_SQNEG_MVT_f32_r(MVT RetVT, Register Op0) {
868 if (RetVT.SimpleTy != MVT::f32)
869 return Register();
870 if ((Subtarget->isNeonAvailable())) {
871 return fastEmitInst_r(MachineInstOpcode: AArch64::SQNEGv1i32, RC: &AArch64::FPR32RegClass, Op0);
872 }
873 return Register();
874}
875
876Register fastEmit_AArch64ISD_SQNEG_MVT_f64_r(MVT RetVT, Register Op0) {
877 if (RetVT.SimpleTy != MVT::f64)
878 return Register();
879 if ((Subtarget->isNeonAvailable())) {
880 return fastEmitInst_r(MachineInstOpcode: AArch64::SQNEGv1i64, RC: &AArch64::FPR64RegClass, Op0);
881 }
882 return Register();
883}
884
885Register fastEmit_AArch64ISD_SQNEG_r(MVT VT, MVT RetVT, Register Op0) {
886 switch (VT.SimpleTy) {
887 case MVT::f32: return fastEmit_AArch64ISD_SQNEG_MVT_f32_r(RetVT, Op0);
888 case MVT::f64: return fastEmit_AArch64ISD_SQNEG_MVT_f64_r(RetVT, Op0);
889 default: return Register();
890 }
891}
892
893// FastEmit functions for AArch64ISD::SUNPKHI.
894
895Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
896 if (RetVT.SimpleTy != MVT::nxv8i16)
897 return Register();
898 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
899 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
900 }
901 return Register();
902}
903
904Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
905 if (RetVT.SimpleTy != MVT::nxv4i32)
906 return Register();
907 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
908 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
909 }
910 return Register();
911}
912
913Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
914 if (RetVT.SimpleTy != MVT::nxv2i64)
915 return Register();
916 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
917 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
918 }
919 return Register();
920}
921
922Register fastEmit_AArch64ISD_SUNPKHI_r(MVT VT, MVT RetVT, Register Op0) {
923 switch (VT.SimpleTy) {
924 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(RetVT, Op0);
925 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(RetVT, Op0);
926 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(RetVT, Op0);
927 default: return Register();
928 }
929}
930
931// FastEmit functions for AArch64ISD::SUNPKLO.
932
933Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
934 if (RetVT.SimpleTy != MVT::nxv8i16)
935 return Register();
936 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
937 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
938 }
939 return Register();
940}
941
942Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
943 if (RetVT.SimpleTy != MVT::nxv4i32)
944 return Register();
945 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
946 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
947 }
948 return Register();
949}
950
951Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
952 if (RetVT.SimpleTy != MVT::nxv2i64)
953 return Register();
954 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
955 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
956 }
957 return Register();
958}
959
960Register fastEmit_AArch64ISD_SUNPKLO_r(MVT VT, MVT RetVT, Register Op0) {
961 switch (VT.SimpleTy) {
962 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(RetVT, Op0);
963 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(RetVT, Op0);
964 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(RetVT, Op0);
965 default: return Register();
966 }
967}
968
969// FastEmit functions for AArch64ISD::UADDLP.
970
971Register fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(MVT RetVT, Register Op0) {
972 if (RetVT.SimpleTy != MVT::v4i16)
973 return Register();
974 if ((Subtarget->isNeonAvailable())) {
975 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv8i8_v4i16, RC: &AArch64::FPR64RegClass, Op0);
976 }
977 return Register();
978}
979
980Register fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(MVT RetVT, Register Op0) {
981 if (RetVT.SimpleTy != MVT::v8i16)
982 return Register();
983 if ((Subtarget->isNeonAvailable())) {
984 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv16i8_v8i16, RC: &AArch64::FPR128RegClass, Op0);
985 }
986 return Register();
987}
988
989Register fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(MVT RetVT, Register Op0) {
990 if (RetVT.SimpleTy != MVT::v2i32)
991 return Register();
992 if ((Subtarget->isNeonAvailable())) {
993 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv4i16_v2i32, RC: &AArch64::FPR64RegClass, Op0);
994 }
995 return Register();
996}
997
998Register fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(MVT RetVT, Register Op0) {
999 if (RetVT.SimpleTy != MVT::v4i32)
1000 return Register();
1001 if ((Subtarget->isNeonAvailable())) {
1002 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv8i16_v4i32, RC: &AArch64::FPR128RegClass, Op0);
1003 }
1004 return Register();
1005}
1006
1007Register fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(MVT RetVT, Register Op0) {
1008 if (RetVT.SimpleTy != MVT::v1i64)
1009 return Register();
1010 if ((Subtarget->isNeonAvailable())) {
1011 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv2i32_v1i64, RC: &AArch64::FPR64RegClass, Op0);
1012 }
1013 return Register();
1014}
1015
1016Register fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(MVT RetVT, Register Op0) {
1017 if (RetVT.SimpleTy != MVT::v2i64)
1018 return Register();
1019 if ((Subtarget->isNeonAvailable())) {
1020 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv4i32_v2i64, RC: &AArch64::FPR128RegClass, Op0);
1021 }
1022 return Register();
1023}
1024
1025Register fastEmit_AArch64ISD_UADDLP_r(MVT VT, MVT RetVT, Register Op0) {
1026 switch (VT.SimpleTy) {
1027 case MVT::v8i8: return fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(RetVT, Op0);
1028 case MVT::v16i8: return fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(RetVT, Op0);
1029 case MVT::v4i16: return fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(RetVT, Op0);
1030 case MVT::v8i16: return fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(RetVT, Op0);
1031 case MVT::v2i32: return fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(RetVT, Op0);
1032 case MVT::v4i32: return fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(RetVT, Op0);
1033 default: return Register();
1034 }
1035}
1036
1037// FastEmit functions for AArch64ISD::UITOF.
1038
1039Register fastEmit_AArch64ISD_UITOF_MVT_f16_r(MVT RetVT, Register Op0) {
1040 if (RetVT.SimpleTy != MVT::f16)
1041 return Register();
1042 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
1043 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i16, RC: &AArch64::FPR16RegClass, Op0);
1044 }
1045 return Register();
1046}
1047
1048Register fastEmit_AArch64ISD_UITOF_MVT_f32_r(MVT RetVT, Register Op0) {
1049 if (RetVT.SimpleTy != MVT::f32)
1050 return Register();
1051 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
1052 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i32, RC: &AArch64::FPR32RegClass, Op0);
1053 }
1054 return Register();
1055}
1056
1057Register fastEmit_AArch64ISD_UITOF_MVT_f64_r(MVT RetVT, Register Op0) {
1058 if (RetVT.SimpleTy != MVT::f64)
1059 return Register();
1060 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
1061 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i64, RC: &AArch64::FPR64RegClass, Op0);
1062 }
1063 return Register();
1064}
1065
1066Register fastEmit_AArch64ISD_UITOF_r(MVT VT, MVT RetVT, Register Op0) {
1067 switch (VT.SimpleTy) {
1068 case MVT::f16: return fastEmit_AArch64ISD_UITOF_MVT_f16_r(RetVT, Op0);
1069 case MVT::f32: return fastEmit_AArch64ISD_UITOF_MVT_f32_r(RetVT, Op0);
1070 case MVT::f64: return fastEmit_AArch64ISD_UITOF_MVT_f64_r(RetVT, Op0);
1071 default: return Register();
1072 }
1073}
1074
1075// FastEmit functions for AArch64ISD::UUNPKHI.
1076
1077Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
1078 if (RetVT.SimpleTy != MVT::nxv8i16)
1079 return Register();
1080 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1081 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1082 }
1083 return Register();
1084}
1085
1086Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
1087 if (RetVT.SimpleTy != MVT::nxv4i32)
1088 return Register();
1089 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1090 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1091 }
1092 return Register();
1093}
1094
1095Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
1096 if (RetVT.SimpleTy != MVT::nxv2i64)
1097 return Register();
1098 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1099 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1100 }
1101 return Register();
1102}
1103
1104Register fastEmit_AArch64ISD_UUNPKHI_r(MVT VT, MVT RetVT, Register Op0) {
1105 switch (VT.SimpleTy) {
1106 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(RetVT, Op0);
1107 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(RetVT, Op0);
1108 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(RetVT, Op0);
1109 default: return Register();
1110 }
1111}
1112
1113// FastEmit functions for AArch64ISD::UUNPKLO.
1114
1115Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
1116 if (RetVT.SimpleTy != MVT::nxv8i16)
1117 return Register();
1118 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1119 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1120 }
1121 return Register();
1122}
1123
1124Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
1125 if (RetVT.SimpleTy != MVT::nxv4i32)
1126 return Register();
1127 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1128 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1129 }
1130 return Register();
1131}
1132
1133Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
1134 if (RetVT.SimpleTy != MVT::nxv2i64)
1135 return Register();
1136 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1137 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1138 }
1139 return Register();
1140}
1141
1142Register fastEmit_AArch64ISD_UUNPKLO_r(MVT VT, MVT RetVT, Register Op0) {
1143 switch (VT.SimpleTy) {
1144 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(RetVT, Op0);
1145 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(RetVT, Op0);
1146 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(RetVT, Op0);
1147 default: return Register();
1148 }
1149}
1150
1151// FastEmit functions for ISD::ABS.
1152
1153Register fastEmit_ISD_ABS_MVT_i32_r(MVT RetVT, Register Op0) {
1154 if (RetVT.SimpleTy != MVT::i32)
1155 return Register();
1156 if ((Subtarget->hasCSSC())) {
1157 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSWr, RC: &AArch64::GPR32RegClass, Op0);
1158 }
1159 return Register();
1160}
1161
1162Register fastEmit_ISD_ABS_MVT_i64_r(MVT RetVT, Register Op0) {
1163 if (RetVT.SimpleTy != MVT::i64)
1164 return Register();
1165 if ((!Subtarget->hasCSSC())) {
1166 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
1167 }
1168 if ((Subtarget->hasCSSC())) {
1169 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSXr, RC: &AArch64::GPR64RegClass, Op0);
1170 }
1171 return Register();
1172}
1173
1174Register fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, Register Op0) {
1175 if (RetVT.SimpleTy != MVT::v8i8)
1176 return Register();
1177 if ((Subtarget->isNeonAvailable())) {
1178 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv8i8, RC: &AArch64::FPR64RegClass, Op0);
1179 }
1180 return Register();
1181}
1182
1183Register fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, Register Op0) {
1184 if (RetVT.SimpleTy != MVT::v16i8)
1185 return Register();
1186 if ((Subtarget->isNeonAvailable())) {
1187 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv16i8, RC: &AArch64::FPR128RegClass, Op0);
1188 }
1189 return Register();
1190}
1191
1192Register fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, Register Op0) {
1193 if (RetVT.SimpleTy != MVT::v4i16)
1194 return Register();
1195 if ((Subtarget->isNeonAvailable())) {
1196 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv4i16, RC: &AArch64::FPR64RegClass, Op0);
1197 }
1198 return Register();
1199}
1200
1201Register fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, Register Op0) {
1202 if (RetVT.SimpleTy != MVT::v8i16)
1203 return Register();
1204 if ((Subtarget->isNeonAvailable())) {
1205 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv8i16, RC: &AArch64::FPR128RegClass, Op0);
1206 }
1207 return Register();
1208}
1209
1210Register fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, Register Op0) {
1211 if (RetVT.SimpleTy != MVT::v2i32)
1212 return Register();
1213 if ((Subtarget->isNeonAvailable())) {
1214 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv2i32, RC: &AArch64::FPR64RegClass, Op0);
1215 }
1216 return Register();
1217}
1218
1219Register fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, Register Op0) {
1220 if (RetVT.SimpleTy != MVT::v4i32)
1221 return Register();
1222 if ((Subtarget->isNeonAvailable())) {
1223 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv4i32, RC: &AArch64::FPR128RegClass, Op0);
1224 }
1225 return Register();
1226}
1227
1228Register fastEmit_ISD_ABS_MVT_v1i64_r(MVT RetVT, Register Op0) {
1229 if (RetVT.SimpleTy != MVT::v1i64)
1230 return Register();
1231 if ((Subtarget->isNeonAvailable())) {
1232 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
1233 }
1234 return Register();
1235}
1236
1237Register fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, Register Op0) {
1238 if (RetVT.SimpleTy != MVT::v2i64)
1239 return Register();
1240 if ((Subtarget->isNeonAvailable())) {
1241 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv2i64, RC: &AArch64::FPR128RegClass, Op0);
1242 }
1243 return Register();
1244}
1245
1246Register fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, Register Op0) {
1247 switch (VT.SimpleTy) {
1248 case MVT::i32: return fastEmit_ISD_ABS_MVT_i32_r(RetVT, Op0);
1249 case MVT::i64: return fastEmit_ISD_ABS_MVT_i64_r(RetVT, Op0);
1250 case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0);
1251 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
1252 case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0);
1253 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
1254 case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0);
1255 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
1256 case MVT::v1i64: return fastEmit_ISD_ABS_MVT_v1i64_r(RetVT, Op0);
1257 case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0);
1258 default: return Register();
1259 }
1260}
1261
1262// FastEmit functions for ISD::BITCAST.
1263
1264Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Register Op0) {
1265 if ((!Subtarget->isLittleEndian())) {
1266 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1267 }
1268 return Register();
1269}
1270
1271Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Register Op0) {
1272 if ((!Subtarget->isLittleEndian())) {
1273 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1274 }
1275 return Register();
1276}
1277
1278Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Register Op0) {
1279 if ((!Subtarget->isLittleEndian())) {
1280 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1281 }
1282 return Register();
1283}
1284
1285Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Register Op0) {
1286 if ((!Subtarget->isLittleEndian())) {
1287 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1288 }
1289 return Register();
1290}
1291
1292Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Register Op0) {
1293 if ((!Subtarget->isLittleEndian())) {
1294 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1295 }
1296 return Register();
1297}
1298
1299Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Register Op0) {
1300 if ((!Subtarget->isLittleEndian())) {
1301 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1302 }
1303 return Register();
1304}
1305
1306Register fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, Register Op0) {
1307switch (RetVT.SimpleTy) {
1308 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0);
1309 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0);
1310 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0);
1311 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0);
1312 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Op0);
1313 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0);
1314 default: return Register();
1315}
1316}
1317
1318Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Register Op0) {
1319 if ((!Subtarget->isLittleEndian())) {
1320 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1321 }
1322 return Register();
1323}
1324
1325Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Register Op0) {
1326 if ((!Subtarget->isLittleEndian())) {
1327 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1328 }
1329 return Register();
1330}
1331
1332Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Register Op0) {
1333 if ((!Subtarget->isLittleEndian())) {
1334 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1335 }
1336 return Register();
1337}
1338
1339Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Register Op0) {
1340 if ((!Subtarget->isLittleEndian())) {
1341 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1342 }
1343 return Register();
1344}
1345
1346Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Register Op0) {
1347 if ((!Subtarget->isLittleEndian())) {
1348 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1349 }
1350 return Register();
1351}
1352
1353Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Register Op0) {
1354 if ((!Subtarget->isLittleEndian())) {
1355 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1356 }
1357 return Register();
1358}
1359
1360Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Register Op0) {
1361 if ((!Subtarget->isLittleEndian())) {
1362 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1363 }
1364 return Register();
1365}
1366
1367Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Register Op0) {
1368 if ((!Subtarget->isLittleEndian())) {
1369 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1370 }
1371 return Register();
1372}
1373
1374Register fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, Register Op0) {
1375switch (RetVT.SimpleTy) {
1376 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0);
1377 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0);
1378 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0);
1379 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0);
1380 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0);
1381 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Op0);
1382 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0);
1383 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Op0);
1384 default: return Register();
1385}
1386}
1387
1388Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Register Op0) {
1389 if ((!Subtarget->isLittleEndian())) {
1390 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1391 }
1392 return Register();
1393}
1394
1395Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Register Op0) {
1396 if ((!Subtarget->isLittleEndian())) {
1397 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1398 }
1399 return Register();
1400}
1401
1402Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Register Op0) {
1403 if ((!Subtarget->isLittleEndian())) {
1404 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1405 }
1406 return Register();
1407}
1408
1409Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Register Op0) {
1410 if ((!Subtarget->isLittleEndian())) {
1411 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1412 }
1413 return Register();
1414}
1415
1416Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Register Op0) {
1417 if ((!Subtarget->isLittleEndian())) {
1418 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1419 }
1420 return Register();
1421}
1422
1423Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Register Op0) {
1424 if ((!Subtarget->isLittleEndian())) {
1425 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1426 }
1427 return Register();
1428}
1429
1430Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Register Op0) {
1431 if ((!Subtarget->isLittleEndian())) {
1432 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1433 }
1434 return Register();
1435}
1436
1437Register fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, Register Op0) {
1438switch (RetVT.SimpleTy) {
1439 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0);
1440 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0);
1441 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0);
1442 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0);
1443 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Op0);
1444 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0);
1445 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0);
1446 default: return Register();
1447}
1448}
1449
1450Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Register Op0) {
1451 if ((!Subtarget->isLittleEndian())) {
1452 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1453 }
1454 return Register();
1455}
1456
1457Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Register Op0) {
1458 if ((!Subtarget->isLittleEndian())) {
1459 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1460 }
1461 return Register();
1462}
1463
1464Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Register Op0) {
1465 if ((!Subtarget->isLittleEndian())) {
1466 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1467 }
1468 return Register();
1469}
1470
1471Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Register Op0) {
1472 if ((!Subtarget->isLittleEndian())) {
1473 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1474 }
1475 return Register();
1476}
1477
1478Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Register Op0) {
1479 if ((!Subtarget->isLittleEndian())) {
1480 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1481 }
1482 return Register();
1483}
1484
1485Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Register Op0) {
1486 if ((!Subtarget->isLittleEndian())) {
1487 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1488 }
1489 return Register();
1490}
1491
1492Register fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, Register Op0) {
1493switch (RetVT.SimpleTy) {
1494 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0);
1495 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0);
1496 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0);
1497 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0);
1498 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0);
1499 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Op0);
1500 default: return Register();
1501}
1502}
1503
1504Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Register Op0) {
1505 if ((!Subtarget->isLittleEndian())) {
1506 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1507 }
1508 return Register();
1509}
1510
1511Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Register Op0) {
1512 if ((!Subtarget->isLittleEndian())) {
1513 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1514 }
1515 return Register();
1516}
1517
1518Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Register Op0) {
1519 if ((!Subtarget->isLittleEndian())) {
1520 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1521 }
1522 return Register();
1523}
1524
1525Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Register Op0) {
1526 if ((!Subtarget->isLittleEndian())) {
1527 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1528 }
1529 return Register();
1530}
1531
1532Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Register Op0) {
1533 if ((!Subtarget->isLittleEndian())) {
1534 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1535 }
1536 return Register();
1537}
1538
1539Register fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, Register Op0) {
1540switch (RetVT.SimpleTy) {
1541 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0);
1542 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0);
1543 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0);
1544 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0);
1545 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0);
1546 default: return Register();
1547}
1548}
1549
1550Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Register Op0) {
1551 if ((!Subtarget->isLittleEndian())) {
1552 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1553 }
1554 return Register();
1555}
1556
1557Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Register Op0) {
1558 if ((!Subtarget->isLittleEndian())) {
1559 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1560 }
1561 return Register();
1562}
1563
1564Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Register Op0) {
1565 if ((!Subtarget->isLittleEndian())) {
1566 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1567 }
1568 return Register();
1569}
1570
1571Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Register Op0) {
1572 if ((!Subtarget->isLittleEndian())) {
1573 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1574 }
1575 return Register();
1576}
1577
1578Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Register Op0) {
1579 if ((!Subtarget->isLittleEndian())) {
1580 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1581 }
1582 return Register();
1583}
1584
1585Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Register Op0) {
1586 if ((!Subtarget->isLittleEndian())) {
1587 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1588 }
1589 return Register();
1590}
1591
1592Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Register Op0) {
1593 if ((!Subtarget->isLittleEndian())) {
1594 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1595 }
1596 return Register();
1597}
1598
1599Register fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, Register Op0) {
1600switch (RetVT.SimpleTy) {
1601 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0);
1602 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0);
1603 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0);
1604 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0);
1605 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0);
1606 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Op0);
1607 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Op0);
1608 default: return Register();
1609}
1610}
1611
1612Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Register Op0) {
1613 if ((!Subtarget->isLittleEndian())) {
1614 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1615 }
1616 return Register();
1617}
1618
1619Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Register Op0) {
1620 if ((!Subtarget->isLittleEndian())) {
1621 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1622 }
1623 return Register();
1624}
1625
1626Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Register Op0) {
1627 if ((!Subtarget->isLittleEndian())) {
1628 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1629 }
1630 return Register();
1631}
1632
1633Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Register Op0) {
1634 if ((!Subtarget->isLittleEndian())) {
1635 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1636 }
1637 return Register();
1638}
1639
1640Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Register Op0) {
1641 if ((!Subtarget->isLittleEndian())) {
1642 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1643 }
1644 return Register();
1645}
1646
1647Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Register Op0) {
1648 if ((!Subtarget->isLittleEndian())) {
1649 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1650 }
1651 return Register();
1652}
1653
1654Register fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, Register Op0) {
1655switch (RetVT.SimpleTy) {
1656 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0);
1657 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0);
1658 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0);
1659 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0);
1660 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Op0);
1661 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0);
1662 default: return Register();
1663}
1664}
1665
1666Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Register Op0) {
1667 if ((!Subtarget->isLittleEndian())) {
1668 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1669 }
1670 return Register();
1671}
1672
1673Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Register Op0) {
1674 if ((!Subtarget->isLittleEndian())) {
1675 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1676 }
1677 return Register();
1678}
1679
1680Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Register Op0) {
1681 if ((!Subtarget->isLittleEndian())) {
1682 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1683 }
1684 return Register();
1685}
1686
1687Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Register Op0) {
1688 if ((!Subtarget->isLittleEndian())) {
1689 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1690 }
1691 return Register();
1692}
1693
1694Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Register Op0) {
1695 if ((!Subtarget->isLittleEndian())) {
1696 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1697 }
1698 return Register();
1699}
1700
1701Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Register Op0) {
1702 if ((!Subtarget->isLittleEndian())) {
1703 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1704 }
1705 return Register();
1706}
1707
1708Register fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, Register Op0) {
1709switch (RetVT.SimpleTy) {
1710 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0);
1711 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0);
1712 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0);
1713 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0);
1714 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Op0);
1715 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0);
1716 default: return Register();
1717}
1718}
1719
1720Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Register Op0) {
1721 if ((!Subtarget->isLittleEndian())) {
1722 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1723 }
1724 return Register();
1725}
1726
1727Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Register Op0) {
1728 if ((!Subtarget->isLittleEndian())) {
1729 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1730 }
1731 return Register();
1732}
1733
1734Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Register Op0) {
1735 if ((!Subtarget->isLittleEndian())) {
1736 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1737 }
1738 return Register();
1739}
1740
1741Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Register Op0) {
1742 if ((!Subtarget->isLittleEndian())) {
1743 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1744 }
1745 return Register();
1746}
1747
1748Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Register Op0) {
1749 if ((!Subtarget->isLittleEndian())) {
1750 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1751 }
1752 return Register();
1753}
1754
1755Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Register Op0) {
1756 if ((!Subtarget->isLittleEndian())) {
1757 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1758 }
1759 return Register();
1760}
1761
1762Register fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, Register Op0) {
1763switch (RetVT.SimpleTy) {
1764 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0);
1765 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0);
1766 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0);
1767 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0);
1768 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Op0);
1769 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0);
1770 default: return Register();
1771}
1772}
1773
1774Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Register Op0) {
1775 if ((!Subtarget->isLittleEndian())) {
1776 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1777 }
1778 return Register();
1779}
1780
1781Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Register Op0) {
1782 if ((!Subtarget->isLittleEndian())) {
1783 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1784 }
1785 return Register();
1786}
1787
1788Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Register Op0) {
1789 if ((!Subtarget->isLittleEndian())) {
1790 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1791 }
1792 return Register();
1793}
1794
1795Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Register Op0) {
1796 if ((!Subtarget->isLittleEndian())) {
1797 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1798 }
1799 return Register();
1800}
1801
1802Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Register Op0) {
1803 if ((!Subtarget->isLittleEndian())) {
1804 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1805 }
1806 return Register();
1807}
1808
1809Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Register Op0) {
1810 if ((!Subtarget->isLittleEndian())) {
1811 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1812 }
1813 return Register();
1814}
1815
1816Register fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, Register Op0) {
1817switch (RetVT.SimpleTy) {
1818 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0);
1819 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0);
1820 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0);
1821 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0);
1822 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0);
1823 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Op0);
1824 default: return Register();
1825}
1826}
1827
1828Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Register Op0) {
1829 if ((!Subtarget->isLittleEndian())) {
1830 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1831 }
1832 return Register();
1833}
1834
1835Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Register Op0) {
1836 if ((!Subtarget->isLittleEndian())) {
1837 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1838 }
1839 return Register();
1840}
1841
1842Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Register Op0) {
1843 if ((!Subtarget->isLittleEndian())) {
1844 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1845 }
1846 return Register();
1847}
1848
1849Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Register Op0) {
1850 if ((!Subtarget->isLittleEndian())) {
1851 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1852 }
1853 return Register();
1854}
1855
1856Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Register Op0) {
1857 if ((!Subtarget->isLittleEndian())) {
1858 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1859 }
1860 return Register();
1861}
1862
1863Register fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, Register Op0) {
1864switch (RetVT.SimpleTy) {
1865 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0);
1866 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0);
1867 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0);
1868 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0);
1869 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0);
1870 default: return Register();
1871}
1872}
1873
1874Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Register Op0) {
1875 if ((!Subtarget->isLittleEndian())) {
1876 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1877 }
1878 return Register();
1879}
1880
1881Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Register Op0) {
1882 if ((!Subtarget->isLittleEndian())) {
1883 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1884 }
1885 return Register();
1886}
1887
1888Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Register Op0) {
1889 if ((!Subtarget->isLittleEndian())) {
1890 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1891 }
1892 return Register();
1893}
1894
1895Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Register Op0) {
1896 if ((!Subtarget->isLittleEndian())) {
1897 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1898 }
1899 return Register();
1900}
1901
1902Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Register Op0) {
1903 if ((!Subtarget->isLittleEndian())) {
1904 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1905 }
1906 return Register();
1907}
1908
1909Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(Register Op0) {
1910 if ((!Subtarget->isLittleEndian())) {
1911 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1912 }
1913 return Register();
1914}
1915
1916Register fastEmit_ISD_BITCAST_MVT_v4bf16_r(MVT RetVT, Register Op0) {
1917switch (RetVT.SimpleTy) {
1918 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Op0);
1919 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Op0);
1920 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Op0);
1921 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Op0);
1922 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Op0);
1923 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(Op0);
1924 default: return Register();
1925}
1926}
1927
1928Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Register Op0) {
1929 if ((!Subtarget->isLittleEndian())) {
1930 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1931 }
1932 return Register();
1933}
1934
1935Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Register Op0) {
1936 if ((!Subtarget->isLittleEndian())) {
1937 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1938 }
1939 return Register();
1940}
1941
1942Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Register Op0) {
1943 if ((!Subtarget->isLittleEndian())) {
1944 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1945 }
1946 return Register();
1947}
1948
1949Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Register Op0) {
1950 if ((!Subtarget->isLittleEndian())) {
1951 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1952 }
1953 return Register();
1954}
1955
1956Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Register Op0) {
1957 if ((!Subtarget->isLittleEndian())) {
1958 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1959 }
1960 return Register();
1961}
1962
1963Register fastEmit_ISD_BITCAST_MVT_v8bf16_r(MVT RetVT, Register Op0) {
1964switch (RetVT.SimpleTy) {
1965 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Op0);
1966 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Op0);
1967 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Op0);
1968 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Op0);
1969 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Op0);
1970 default: return Register();
1971}
1972}
1973
1974Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Register Op0) {
1975 if ((!Subtarget->isLittleEndian())) {
1976 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1977 }
1978 return Register();
1979}
1980
1981Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Register Op0) {
1982 if ((!Subtarget->isLittleEndian())) {
1983 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1984 }
1985 return Register();
1986}
1987
1988Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Register Op0) {
1989 if ((!Subtarget->isLittleEndian())) {
1990 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1991 }
1992 return Register();
1993}
1994
1995Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Register Op0) {
1996 if ((!Subtarget->isLittleEndian())) {
1997 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1998 }
1999 return Register();
2000}
2001
2002Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Register Op0) {
2003 if ((!Subtarget->isLittleEndian())) {
2004 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2005 }
2006 return Register();
2007}
2008
2009Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Register Op0) {
2010 if ((!Subtarget->isLittleEndian())) {
2011 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2012 }
2013 return Register();
2014}
2015
2016Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Register Op0) {
2017 if ((!Subtarget->isLittleEndian())) {
2018 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2019 }
2020 return Register();
2021}
2022
2023Register fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, Register Op0) {
2024switch (RetVT.SimpleTy) {
2025 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0);
2026 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0);
2027 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0);
2028 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0);
2029 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0);
2030 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Op0);
2031 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Op0);
2032 default: return Register();
2033}
2034}
2035
2036Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Register Op0) {
2037 if ((!Subtarget->isLittleEndian())) {
2038 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
2039 }
2040 return Register();
2041}
2042
2043Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Register Op0) {
2044 if ((!Subtarget->isLittleEndian())) {
2045 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2046 }
2047 return Register();
2048}
2049
2050Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Register Op0) {
2051 if ((!Subtarget->isLittleEndian())) {
2052 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2053 }
2054 return Register();
2055}
2056
2057Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Register Op0) {
2058 if ((!Subtarget->isLittleEndian())) {
2059 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2060 }
2061 return Register();
2062}
2063
2064Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Register Op0) {
2065 if ((!Subtarget->isLittleEndian())) {
2066 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2067 }
2068 return Register();
2069}
2070
2071Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Register Op0) {
2072 if ((!Subtarget->isLittleEndian())) {
2073 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2074 }
2075 return Register();
2076}
2077
2078Register fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, Register Op0) {
2079switch (RetVT.SimpleTy) {
2080 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0);
2081 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0);
2082 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0);
2083 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0);
2084 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Op0);
2085 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0);
2086 default: return Register();
2087}
2088}
2089
2090Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Register Op0) {
2091 if ((!Subtarget->isLittleEndian())) {
2092 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
2093 }
2094 return Register();
2095}
2096
2097Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Register Op0) {
2098 if ((!Subtarget->isLittleEndian())) {
2099 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2100 }
2101 return Register();
2102}
2103
2104Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Register Op0) {
2105 if ((!Subtarget->isLittleEndian())) {
2106 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2107 }
2108 return Register();
2109}
2110
2111Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Register Op0) {
2112 if ((!Subtarget->isLittleEndian())) {
2113 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2114 }
2115 return Register();
2116}
2117
2118Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(Register Op0) {
2119 if ((!Subtarget->isLittleEndian())) {
2120 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2121 }
2122 return Register();
2123}
2124
2125Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Register Op0) {
2126 if ((!Subtarget->isLittleEndian())) {
2127 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2128 }
2129 return Register();
2130}
2131
2132Register fastEmit_ISD_BITCAST_MVT_v1f64_r(MVT RetVT, Register Op0) {
2133switch (RetVT.SimpleTy) {
2134 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Op0);
2135 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Op0);
2136 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Op0);
2137 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Op0);
2138 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(Op0);
2139 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Op0);
2140 default: return Register();
2141}
2142}
2143
2144Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Register Op0) {
2145 if ((!Subtarget->isLittleEndian())) {
2146 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2147 }
2148 return Register();
2149}
2150
2151Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Register Op0) {
2152 if ((!Subtarget->isLittleEndian())) {
2153 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2154 }
2155 return Register();
2156}
2157
2158Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Register Op0) {
2159 if ((!Subtarget->isLittleEndian())) {
2160 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2161 }
2162 return Register();
2163}
2164
2165Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Register Op0) {
2166 if ((!Subtarget->isLittleEndian())) {
2167 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2168 }
2169 return Register();
2170}
2171
2172Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Register Op0) {
2173 if ((!Subtarget->isLittleEndian())) {
2174 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2175 }
2176 return Register();
2177}
2178
2179Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Register Op0) {
2180 if ((!Subtarget->isLittleEndian())) {
2181 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2182 }
2183 return Register();
2184}
2185
2186Register fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, Register Op0) {
2187switch (RetVT.SimpleTy) {
2188 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0);
2189 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0);
2190 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0);
2191 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0);
2192 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Op0);
2193 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0);
2194 default: return Register();
2195}
2196}
2197
2198Register fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, Register Op0) {
2199 switch (VT.SimpleTy) {
2200 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
2201 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0);
2202 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0);
2203 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0);
2204 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0);
2205 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0);
2206 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0);
2207 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0);
2208 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0);
2209 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0);
2210 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0);
2211 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v4bf16_r(RetVT, Op0);
2212 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v8bf16_r(RetVT, Op0);
2213 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0);
2214 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0);
2215 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v1f64_r(RetVT, Op0);
2216 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0);
2217 default: return Register();
2218 }
2219}
2220
2221// FastEmit functions for ISD::BITREVERSE.
2222
2223Register fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, Register Op0) {
2224 if (RetVT.SimpleTy != MVT::i32)
2225 return Register();
2226 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITWr, RC: &AArch64::GPR32RegClass, Op0);
2227}
2228
2229Register fastEmit_ISD_BITREVERSE_MVT_i64_r(MVT RetVT, Register Op0) {
2230 if (RetVT.SimpleTy != MVT::i64)
2231 return Register();
2232 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITXr, RC: &AArch64::GPR64RegClass, Op0);
2233}
2234
2235Register fastEmit_ISD_BITREVERSE_MVT_v8i8_r(MVT RetVT, Register Op0) {
2236 if (RetVT.SimpleTy != MVT::v8i8)
2237 return Register();
2238 if ((Subtarget->isNeonAvailable())) {
2239 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITv8i8, RC: &AArch64::FPR64RegClass, Op0);
2240 }
2241 return Register();
2242}
2243
2244Register fastEmit_ISD_BITREVERSE_MVT_v16i8_r(MVT RetVT, Register Op0) {
2245 if (RetVT.SimpleTy != MVT::v16i8)
2246 return Register();
2247 if ((Subtarget->isNeonAvailable())) {
2248 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITv16i8, RC: &AArch64::FPR128RegClass, Op0);
2249 }
2250 return Register();
2251}
2252
2253Register fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, Register Op0) {
2254 switch (VT.SimpleTy) {
2255 case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0);
2256 case MVT::i64: return fastEmit_ISD_BITREVERSE_MVT_i64_r(RetVT, Op0);
2257 case MVT::v8i8: return fastEmit_ISD_BITREVERSE_MVT_v8i8_r(RetVT, Op0);
2258 case MVT::v16i8: return fastEmit_ISD_BITREVERSE_MVT_v16i8_r(RetVT, Op0);
2259 default: return Register();
2260 }
2261}
2262
2263// FastEmit functions for ISD::BRIND.
2264
2265Register fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, Register Op0) {
2266 if (RetVT.SimpleTy != MVT::isVoid)
2267 return Register();
2268 return fastEmitInst_r(MachineInstOpcode: AArch64::BR, RC: &AArch64::GPR64RegClass, Op0);
2269}
2270
2271Register fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, Register Op0) {
2272 switch (VT.SimpleTy) {
2273 case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0);
2274 default: return Register();
2275 }
2276}
2277
2278// FastEmit functions for ISD::BSWAP.
2279
2280Register fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, Register Op0) {
2281 if (RetVT.SimpleTy != MVT::i32)
2282 return Register();
2283 return fastEmitInst_r(MachineInstOpcode: AArch64::REVWr, RC: &AArch64::GPR32RegClass, Op0);
2284}
2285
2286Register fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, Register Op0) {
2287 if (RetVT.SimpleTy != MVT::i64)
2288 return Register();
2289 return fastEmitInst_r(MachineInstOpcode: AArch64::REVXr, RC: &AArch64::GPR64RegClass, Op0);
2290}
2291
2292Register fastEmit_ISD_BSWAP_MVT_v4i16_r(MVT RetVT, Register Op0) {
2293 if (RetVT.SimpleTy != MVT::v4i16)
2294 return Register();
2295 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
2296}
2297
2298Register fastEmit_ISD_BSWAP_MVT_v8i16_r(MVT RetVT, Register Op0) {
2299 if (RetVT.SimpleTy != MVT::v8i16)
2300 return Register();
2301 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
2302}
2303
2304Register fastEmit_ISD_BSWAP_MVT_v2i32_r(MVT RetVT, Register Op0) {
2305 if (RetVT.SimpleTy != MVT::v2i32)
2306 return Register();
2307 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
2308}
2309
2310Register fastEmit_ISD_BSWAP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2311 if (RetVT.SimpleTy != MVT::v4i32)
2312 return Register();
2313 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
2314}
2315
2316Register fastEmit_ISD_BSWAP_MVT_v2i64_r(MVT RetVT, Register Op0) {
2317 if (RetVT.SimpleTy != MVT::v2i64)
2318 return Register();
2319 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2320}
2321
2322Register fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, Register Op0) {
2323 switch (VT.SimpleTy) {
2324 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
2325 case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0);
2326 case MVT::v4i16: return fastEmit_ISD_BSWAP_MVT_v4i16_r(RetVT, Op0);
2327 case MVT::v8i16: return fastEmit_ISD_BSWAP_MVT_v8i16_r(RetVT, Op0);
2328 case MVT::v2i32: return fastEmit_ISD_BSWAP_MVT_v2i32_r(RetVT, Op0);
2329 case MVT::v4i32: return fastEmit_ISD_BSWAP_MVT_v4i32_r(RetVT, Op0);
2330 case MVT::v2i64: return fastEmit_ISD_BSWAP_MVT_v2i64_r(RetVT, Op0);
2331 default: return Register();
2332 }
2333}
2334
2335// FastEmit functions for ISD::CTLS.
2336
2337Register fastEmit_ISD_CTLS_MVT_i32_r(MVT RetVT, Register Op0) {
2338 if (RetVT.SimpleTy != MVT::i32)
2339 return Register();
2340 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSWr, RC: &AArch64::GPR32RegClass, Op0);
2341}
2342
2343Register fastEmit_ISD_CTLS_MVT_i64_r(MVT RetVT, Register Op0) {
2344 if (RetVT.SimpleTy != MVT::i64)
2345 return Register();
2346 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSXr, RC: &AArch64::GPR64RegClass, Op0);
2347}
2348
2349Register fastEmit_ISD_CTLS_MVT_v8i8_r(MVT RetVT, Register Op0) {
2350 if (RetVT.SimpleTy != MVT::v8i8)
2351 return Register();
2352 if ((Subtarget->isNeonAvailable())) {
2353 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv8i8, RC: &AArch64::FPR64RegClass, Op0);
2354 }
2355 return Register();
2356}
2357
2358Register fastEmit_ISD_CTLS_MVT_v16i8_r(MVT RetVT, Register Op0) {
2359 if (RetVT.SimpleTy != MVT::v16i8)
2360 return Register();
2361 if ((Subtarget->isNeonAvailable())) {
2362 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv16i8, RC: &AArch64::FPR128RegClass, Op0);
2363 }
2364 return Register();
2365}
2366
2367Register fastEmit_ISD_CTLS_MVT_v4i16_r(MVT RetVT, Register Op0) {
2368 if (RetVT.SimpleTy != MVT::v4i16)
2369 return Register();
2370 if ((Subtarget->isNeonAvailable())) {
2371 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv4i16, RC: &AArch64::FPR64RegClass, Op0);
2372 }
2373 return Register();
2374}
2375
2376Register fastEmit_ISD_CTLS_MVT_v8i16_r(MVT RetVT, Register Op0) {
2377 if (RetVT.SimpleTy != MVT::v8i16)
2378 return Register();
2379 if ((Subtarget->isNeonAvailable())) {
2380 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv8i16, RC: &AArch64::FPR128RegClass, Op0);
2381 }
2382 return Register();
2383}
2384
2385Register fastEmit_ISD_CTLS_MVT_v2i32_r(MVT RetVT, Register Op0) {
2386 if (RetVT.SimpleTy != MVT::v2i32)
2387 return Register();
2388 if ((Subtarget->isNeonAvailable())) {
2389 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv2i32, RC: &AArch64::FPR64RegClass, Op0);
2390 }
2391 return Register();
2392}
2393
2394Register fastEmit_ISD_CTLS_MVT_v4i32_r(MVT RetVT, Register Op0) {
2395 if (RetVT.SimpleTy != MVT::v4i32)
2396 return Register();
2397 if ((Subtarget->isNeonAvailable())) {
2398 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv4i32, RC: &AArch64::FPR128RegClass, Op0);
2399 }
2400 return Register();
2401}
2402
2403Register fastEmit_ISD_CTLS_r(MVT VT, MVT RetVT, Register Op0) {
2404 switch (VT.SimpleTy) {
2405 case MVT::i32: return fastEmit_ISD_CTLS_MVT_i32_r(RetVT, Op0);
2406 case MVT::i64: return fastEmit_ISD_CTLS_MVT_i64_r(RetVT, Op0);
2407 case MVT::v8i8: return fastEmit_ISD_CTLS_MVT_v8i8_r(RetVT, Op0);
2408 case MVT::v16i8: return fastEmit_ISD_CTLS_MVT_v16i8_r(RetVT, Op0);
2409 case MVT::v4i16: return fastEmit_ISD_CTLS_MVT_v4i16_r(RetVT, Op0);
2410 case MVT::v8i16: return fastEmit_ISD_CTLS_MVT_v8i16_r(RetVT, Op0);
2411 case MVT::v2i32: return fastEmit_ISD_CTLS_MVT_v2i32_r(RetVT, Op0);
2412 case MVT::v4i32: return fastEmit_ISD_CTLS_MVT_v4i32_r(RetVT, Op0);
2413 default: return Register();
2414 }
2415}
2416
2417// FastEmit functions for ISD::CTLZ.
2418
2419Register fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, Register Op0) {
2420 if (RetVT.SimpleTy != MVT::i32)
2421 return Register();
2422 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZWr, RC: &AArch64::GPR32RegClass, Op0);
2423}
2424
2425Register fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, Register Op0) {
2426 if (RetVT.SimpleTy != MVT::i64)
2427 return Register();
2428 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZXr, RC: &AArch64::GPR64RegClass, Op0);
2429}
2430
2431Register fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, Register Op0) {
2432 if (RetVT.SimpleTy != MVT::v8i8)
2433 return Register();
2434 if ((Subtarget->isNeonAvailable())) {
2435 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv8i8, RC: &AArch64::FPR64RegClass, Op0);
2436 }
2437 return Register();
2438}
2439
2440Register fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, Register Op0) {
2441 if (RetVT.SimpleTy != MVT::v16i8)
2442 return Register();
2443 if ((Subtarget->isNeonAvailable())) {
2444 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv16i8, RC: &AArch64::FPR128RegClass, Op0);
2445 }
2446 return Register();
2447}
2448
2449Register fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, Register Op0) {
2450 if (RetVT.SimpleTy != MVT::v4i16)
2451 return Register();
2452 if ((Subtarget->isNeonAvailable())) {
2453 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv4i16, RC: &AArch64::FPR64RegClass, Op0);
2454 }
2455 return Register();
2456}
2457
2458Register fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, Register Op0) {
2459 if (RetVT.SimpleTy != MVT::v8i16)
2460 return Register();
2461 if ((Subtarget->isNeonAvailable())) {
2462 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv8i16, RC: &AArch64::FPR128RegClass, Op0);
2463 }
2464 return Register();
2465}
2466
2467Register fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, Register Op0) {
2468 if (RetVT.SimpleTy != MVT::v2i32)
2469 return Register();
2470 if ((Subtarget->isNeonAvailable())) {
2471 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv2i32, RC: &AArch64::FPR64RegClass, Op0);
2472 }
2473 return Register();
2474}
2475
2476Register fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, Register Op0) {
2477 if (RetVT.SimpleTy != MVT::v4i32)
2478 return Register();
2479 if ((Subtarget->isNeonAvailable())) {
2480 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv4i32, RC: &AArch64::FPR128RegClass, Op0);
2481 }
2482 return Register();
2483}
2484
2485Register fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, Register Op0) {
2486 switch (VT.SimpleTy) {
2487 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
2488 case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0);
2489 case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0);
2490 case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0);
2491 case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0);
2492 case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0);
2493 case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0);
2494 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
2495 default: return Register();
2496 }
2497}
2498
2499// FastEmit functions for ISD::CTPOP.
2500
2501Register fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, Register Op0) {
2502 if (RetVT.SimpleTy != MVT::i32)
2503 return Register();
2504 if ((Subtarget->hasCSSC())) {
2505 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTWr, RC: &AArch64::GPR32RegClass, Op0);
2506 }
2507 return Register();
2508}
2509
2510Register fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, Register Op0) {
2511 if (RetVT.SimpleTy != MVT::i64)
2512 return Register();
2513 if ((Subtarget->hasCSSC())) {
2514 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTXr, RC: &AArch64::GPR64RegClass, Op0);
2515 }
2516 return Register();
2517}
2518
2519Register fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, Register Op0) {
2520 if (RetVT.SimpleTy != MVT::v8i8)
2521 return Register();
2522 if ((Subtarget->isNeonAvailable())) {
2523 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTv8i8, RC: &AArch64::FPR64RegClass, Op0);
2524 }
2525 return Register();
2526}
2527
2528Register fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, Register Op0) {
2529 if (RetVT.SimpleTy != MVT::v16i8)
2530 return Register();
2531 if ((Subtarget->isNeonAvailable())) {
2532 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTv16i8, RC: &AArch64::FPR128RegClass, Op0);
2533 }
2534 return Register();
2535}
2536
2537Register fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, Register Op0) {
2538 switch (VT.SimpleTy) {
2539 case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0);
2540 case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0);
2541 case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0);
2542 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
2543 default: return Register();
2544 }
2545}
2546
2547// FastEmit functions for ISD::CTTZ.
2548
2549Register fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, Register Op0) {
2550 if (RetVT.SimpleTy != MVT::i32)
2551 return Register();
2552 if ((Subtarget->hasCSSC())) {
2553 return fastEmitInst_r(MachineInstOpcode: AArch64::CTZWr, RC: &AArch64::GPR32RegClass, Op0);
2554 }
2555 return Register();
2556}
2557
2558Register fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, Register Op0) {
2559 if (RetVT.SimpleTy != MVT::i64)
2560 return Register();
2561 if ((Subtarget->hasCSSC())) {
2562 return fastEmitInst_r(MachineInstOpcode: AArch64::CTZXr, RC: &AArch64::GPR64RegClass, Op0);
2563 }
2564 return Register();
2565}
2566
2567Register fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, Register Op0) {
2568 switch (VT.SimpleTy) {
2569 case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0);
2570 case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0);
2571 default: return Register();
2572 }
2573}
2574
2575// FastEmit functions for ISD::FABS.
2576
2577Register fastEmit_ISD_FABS_MVT_bf16_r(MVT RetVT, Register Op0) {
2578 if (RetVT.SimpleTy != MVT::bf16)
2579 return Register();
2580 if ((Subtarget->hasFullFP16())) {
2581 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSHr, RC: &AArch64::FPR16RegClass, Op0);
2582 }
2583 return Register();
2584}
2585
2586Register fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, Register Op0) {
2587 if (RetVT.SimpleTy != MVT::f16)
2588 return Register();
2589 if ((Subtarget->hasFullFP16())) {
2590 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSHr, RC: &AArch64::FPR16RegClass, Op0);
2591 }
2592 return Register();
2593}
2594
2595Register fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, Register Op0) {
2596 if (RetVT.SimpleTy != MVT::f32)
2597 return Register();
2598 if ((Subtarget->hasFPARMv8())) {
2599 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSSr, RC: &AArch64::FPR32RegClass, Op0);
2600 }
2601 return Register();
2602}
2603
2604Register fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, Register Op0) {
2605 if (RetVT.SimpleTy != MVT::f64)
2606 return Register();
2607 if ((Subtarget->hasFPARMv8())) {
2608 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSDr, RC: &AArch64::FPR64RegClass, Op0);
2609 }
2610 return Register();
2611}
2612
2613Register fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, Register Op0) {
2614 if (RetVT.SimpleTy != MVT::v4f16)
2615 return Register();
2616 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2617 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f16, RC: &AArch64::FPR64RegClass, Op0);
2618 }
2619 return Register();
2620}
2621
2622Register fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, Register Op0) {
2623 if (RetVT.SimpleTy != MVT::v8f16)
2624 return Register();
2625 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2626 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv8f16, RC: &AArch64::FPR128RegClass, Op0);
2627 }
2628 return Register();
2629}
2630
2631Register fastEmit_ISD_FABS_MVT_v4bf16_r(MVT RetVT, Register Op0) {
2632 if (RetVT.SimpleTy != MVT::v4bf16)
2633 return Register();
2634 if ((Subtarget->hasFullFP16())) {
2635 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f16, RC: &AArch64::FPR64RegClass, Op0);
2636 }
2637 return Register();
2638}
2639
2640Register fastEmit_ISD_FABS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
2641 if (RetVT.SimpleTy != MVT::v8bf16)
2642 return Register();
2643 if ((Subtarget->hasFullFP16())) {
2644 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv8f16, RC: &AArch64::FPR128RegClass, Op0);
2645 }
2646 return Register();
2647}
2648
2649Register fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, Register Op0) {
2650 if (RetVT.SimpleTy != MVT::v2f32)
2651 return Register();
2652 if ((Subtarget->isNeonAvailable())) {
2653 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv2f32, RC: &AArch64::FPR64RegClass, Op0);
2654 }
2655 return Register();
2656}
2657
2658Register fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, Register Op0) {
2659 if (RetVT.SimpleTy != MVT::v4f32)
2660 return Register();
2661 if ((Subtarget->isNeonAvailable())) {
2662 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f32, RC: &AArch64::FPR128RegClass, Op0);
2663 }
2664 return Register();
2665}
2666
2667Register fastEmit_ISD_FABS_MVT_v2f64_r(MVT RetVT, Register Op0) {
2668 if (RetVT.SimpleTy != MVT::v2f64)
2669 return Register();
2670 if ((Subtarget->isNeonAvailable())) {
2671 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv2f64, RC: &AArch64::FPR128RegClass, Op0);
2672 }
2673 return Register();
2674}
2675
2676Register fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, Register Op0) {
2677 switch (VT.SimpleTy) {
2678 case MVT::bf16: return fastEmit_ISD_FABS_MVT_bf16_r(RetVT, Op0);
2679 case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0);
2680 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
2681 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
2682 case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0);
2683 case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0);
2684 case MVT::v4bf16: return fastEmit_ISD_FABS_MVT_v4bf16_r(RetVT, Op0);
2685 case MVT::v8bf16: return fastEmit_ISD_FABS_MVT_v8bf16_r(RetVT, Op0);
2686 case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0);
2687 case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0);
2688 case MVT::v2f64: return fastEmit_ISD_FABS_MVT_v2f64_r(RetVT, Op0);
2689 default: return Register();
2690 }
2691}
2692
2693// FastEmit functions for ISD::FCEIL.
2694
2695Register fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
2696 if (RetVT.SimpleTy != MVT::f16)
2697 return Register();
2698 if ((Subtarget->hasFullFP16())) {
2699 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPHr, RC: &AArch64::FPR16RegClass, Op0);
2700 }
2701 return Register();
2702}
2703
2704Register fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
2705 if (RetVT.SimpleTy != MVT::f32)
2706 return Register();
2707 if ((Subtarget->hasFPARMv8())) {
2708 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPSr, RC: &AArch64::FPR32RegClass, Op0);
2709 }
2710 return Register();
2711}
2712
2713Register fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
2714 if (RetVT.SimpleTy != MVT::f64)
2715 return Register();
2716 if ((Subtarget->hasFPARMv8())) {
2717 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPDr, RC: &AArch64::FPR64RegClass, Op0);
2718 }
2719 return Register();
2720}
2721
2722Register fastEmit_ISD_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
2723 if (RetVT.SimpleTy != MVT::v4f16)
2724 return Register();
2725 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2726 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f16, RC: &AArch64::FPR64RegClass, Op0);
2727 }
2728 return Register();
2729}
2730
2731Register fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
2732 if (RetVT.SimpleTy != MVT::v8f16)
2733 return Register();
2734 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2735 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv8f16, RC: &AArch64::FPR128RegClass, Op0);
2736 }
2737 return Register();
2738}
2739
2740Register fastEmit_ISD_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
2741 if (RetVT.SimpleTy != MVT::v2f32)
2742 return Register();
2743 if ((Subtarget->isNeonAvailable())) {
2744 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f32, RC: &AArch64::FPR64RegClass, Op0);
2745 }
2746 return Register();
2747}
2748
2749Register fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
2750 if (RetVT.SimpleTy != MVT::v4f32)
2751 return Register();
2752 if ((Subtarget->isNeonAvailable())) {
2753 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f32, RC: &AArch64::FPR128RegClass, Op0);
2754 }
2755 return Register();
2756}
2757
2758Register fastEmit_ISD_FCEIL_MVT_v2f64_r(MVT RetVT, Register Op0) {
2759 if (RetVT.SimpleTy != MVT::v2f64)
2760 return Register();
2761 if ((Subtarget->isNeonAvailable())) {
2762 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f64, RC: &AArch64::FPR128RegClass, Op0);
2763 }
2764 return Register();
2765}
2766
2767Register fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
2768 switch (VT.SimpleTy) {
2769 case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0);
2770 case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0);
2771 case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0);
2772 case MVT::v4f16: return fastEmit_ISD_FCEIL_MVT_v4f16_r(RetVT, Op0);
2773 case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0);
2774 case MVT::v2f32: return fastEmit_ISD_FCEIL_MVT_v2f32_r(RetVT, Op0);
2775 case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0);
2776 case MVT::v2f64: return fastEmit_ISD_FCEIL_MVT_v2f64_r(RetVT, Op0);
2777 default: return Register();
2778 }
2779}
2780
2781// FastEmit functions for ISD::FFLOOR.
2782
2783Register fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
2784 if (RetVT.SimpleTy != MVT::f16)
2785 return Register();
2786 if ((Subtarget->hasFullFP16())) {
2787 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMHr, RC: &AArch64::FPR16RegClass, Op0);
2788 }
2789 return Register();
2790}
2791
2792Register fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
2793 if (RetVT.SimpleTy != MVT::f32)
2794 return Register();
2795 if ((Subtarget->hasFPARMv8())) {
2796 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMSr, RC: &AArch64::FPR32RegClass, Op0);
2797 }
2798 return Register();
2799}
2800
2801Register fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
2802 if (RetVT.SimpleTy != MVT::f64)
2803 return Register();
2804 if ((Subtarget->hasFPARMv8())) {
2805 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMDr, RC: &AArch64::FPR64RegClass, Op0);
2806 }
2807 return Register();
2808}
2809
2810Register fastEmit_ISD_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
2811 if (RetVT.SimpleTy != MVT::v4f16)
2812 return Register();
2813 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2814 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f16, RC: &AArch64::FPR64RegClass, Op0);
2815 }
2816 return Register();
2817}
2818
2819Register fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
2820 if (RetVT.SimpleTy != MVT::v8f16)
2821 return Register();
2822 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2823 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv8f16, RC: &AArch64::FPR128RegClass, Op0);
2824 }
2825 return Register();
2826}
2827
2828Register fastEmit_ISD_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
2829 if (RetVT.SimpleTy != MVT::v2f32)
2830 return Register();
2831 if ((Subtarget->isNeonAvailable())) {
2832 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f32, RC: &AArch64::FPR64RegClass, Op0);
2833 }
2834 return Register();
2835}
2836
2837Register fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
2838 if (RetVT.SimpleTy != MVT::v4f32)
2839 return Register();
2840 if ((Subtarget->isNeonAvailable())) {
2841 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f32, RC: &AArch64::FPR128RegClass, Op0);
2842 }
2843 return Register();
2844}
2845
2846Register fastEmit_ISD_FFLOOR_MVT_v2f64_r(MVT RetVT, Register Op0) {
2847 if (RetVT.SimpleTy != MVT::v2f64)
2848 return Register();
2849 if ((Subtarget->isNeonAvailable())) {
2850 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f64, RC: &AArch64::FPR128RegClass, Op0);
2851 }
2852 return Register();
2853}
2854
2855Register fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
2856 switch (VT.SimpleTy) {
2857 case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0);
2858 case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0);
2859 case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0);
2860 case MVT::v4f16: return fastEmit_ISD_FFLOOR_MVT_v4f16_r(RetVT, Op0);
2861 case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0);
2862 case MVT::v2f32: return fastEmit_ISD_FFLOOR_MVT_v2f32_r(RetVT, Op0);
2863 case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0);
2864 case MVT::v2f64: return fastEmit_ISD_FFLOOR_MVT_v2f64_r(RetVT, Op0);
2865 default: return Register();
2866 }
2867}
2868
2869// FastEmit functions for ISD::FNEARBYINT.
2870
2871Register fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
2872 if (RetVT.SimpleTy != MVT::f16)
2873 return Register();
2874 if ((Subtarget->hasFullFP16())) {
2875 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIHr, RC: &AArch64::FPR16RegClass, Op0);
2876 }
2877 return Register();
2878}
2879
2880Register fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
2881 if (RetVT.SimpleTy != MVT::f32)
2882 return Register();
2883 if ((Subtarget->hasFPARMv8())) {
2884 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTISr, RC: &AArch64::FPR32RegClass, Op0);
2885 }
2886 return Register();
2887}
2888
2889Register fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
2890 if (RetVT.SimpleTy != MVT::f64)
2891 return Register();
2892 if ((Subtarget->hasFPARMv8())) {
2893 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIDr, RC: &AArch64::FPR64RegClass, Op0);
2894 }
2895 return Register();
2896}
2897
2898Register fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2899 if (RetVT.SimpleTy != MVT::v4f16)
2900 return Register();
2901 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2902 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f16, RC: &AArch64::FPR64RegClass, Op0);
2903 }
2904 return Register();
2905}
2906
2907Register fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2908 if (RetVT.SimpleTy != MVT::v8f16)
2909 return Register();
2910 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2911 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv8f16, RC: &AArch64::FPR128RegClass, Op0);
2912 }
2913 return Register();
2914}
2915
2916Register fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2917 if (RetVT.SimpleTy != MVT::v2f32)
2918 return Register();
2919 if ((Subtarget->isNeonAvailable())) {
2920 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f32, RC: &AArch64::FPR64RegClass, Op0);
2921 }
2922 return Register();
2923}
2924
2925Register fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2926 if (RetVT.SimpleTy != MVT::v4f32)
2927 return Register();
2928 if ((Subtarget->isNeonAvailable())) {
2929 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f32, RC: &AArch64::FPR128RegClass, Op0);
2930 }
2931 return Register();
2932}
2933
2934Register fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
2935 if (RetVT.SimpleTy != MVT::v2f64)
2936 return Register();
2937 if ((Subtarget->isNeonAvailable())) {
2938 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f64, RC: &AArch64::FPR128RegClass, Op0);
2939 }
2940 return Register();
2941}
2942
2943Register fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
2944 switch (VT.SimpleTy) {
2945 case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0);
2946 case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0);
2947 case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0);
2948 case MVT::v4f16: return fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(RetVT, Op0);
2949 case MVT::v8f16: return fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(RetVT, Op0);
2950 case MVT::v2f32: return fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(RetVT, Op0);
2951 case MVT::v4f32: return fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(RetVT, Op0);
2952 case MVT::v2f64: return fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(RetVT, Op0);
2953 default: return Register();
2954 }
2955}
2956
2957// FastEmit functions for ISD::FNEG.
2958
2959Register fastEmit_ISD_FNEG_MVT_bf16_r(MVT RetVT, Register Op0) {
2960 if (RetVT.SimpleTy != MVT::bf16)
2961 return Register();
2962 if ((Subtarget->hasFullFP16())) {
2963 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGHr, RC: &AArch64::FPR16RegClass, Op0);
2964 }
2965 return Register();
2966}
2967
2968Register fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, Register Op0) {
2969 if (RetVT.SimpleTy != MVT::f16)
2970 return Register();
2971 if ((Subtarget->hasFullFP16())) {
2972 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGHr, RC: &AArch64::FPR16RegClass, Op0);
2973 }
2974 return Register();
2975}
2976
2977Register fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, Register Op0) {
2978 if (RetVT.SimpleTy != MVT::f32)
2979 return Register();
2980 if ((Subtarget->hasFPARMv8())) {
2981 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGSr, RC: &AArch64::FPR32RegClass, Op0);
2982 }
2983 return Register();
2984}
2985
2986Register fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, Register Op0) {
2987 if (RetVT.SimpleTy != MVT::f64)
2988 return Register();
2989 if ((Subtarget->hasFPARMv8())) {
2990 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGDr, RC: &AArch64::FPR64RegClass, Op0);
2991 }
2992 return Register();
2993}
2994
2995Register fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, Register Op0) {
2996 if (RetVT.SimpleTy != MVT::v4f16)
2997 return Register();
2998 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2999 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f16, RC: &AArch64::FPR64RegClass, Op0);
3000 }
3001 return Register();
3002}
3003
3004Register fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, Register Op0) {
3005 if (RetVT.SimpleTy != MVT::v8f16)
3006 return Register();
3007 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3008 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv8f16, RC: &AArch64::FPR128RegClass, Op0);
3009 }
3010 return Register();
3011}
3012
3013Register fastEmit_ISD_FNEG_MVT_v4bf16_r(MVT RetVT, Register Op0) {
3014 if (RetVT.SimpleTy != MVT::v4bf16)
3015 return Register();
3016 if ((Subtarget->hasFullFP16())) {
3017 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f16, RC: &AArch64::FPR64RegClass, Op0);
3018 }
3019 return Register();
3020}
3021
3022Register fastEmit_ISD_FNEG_MVT_v8bf16_r(MVT RetVT, Register Op0) {
3023 if (RetVT.SimpleTy != MVT::v8bf16)
3024 return Register();
3025 if ((Subtarget->hasFullFP16())) {
3026 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv8f16, RC: &AArch64::FPR128RegClass, Op0);
3027 }
3028 return Register();
3029}
3030
3031Register fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, Register Op0) {
3032 if (RetVT.SimpleTy != MVT::v2f32)
3033 return Register();
3034 if ((Subtarget->isNeonAvailable())) {
3035 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv2f32, RC: &AArch64::FPR64RegClass, Op0);
3036 }
3037 return Register();
3038}
3039
3040Register fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, Register Op0) {
3041 if (RetVT.SimpleTy != MVT::v4f32)
3042 return Register();
3043 if ((Subtarget->isNeonAvailable())) {
3044 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f32, RC: &AArch64::FPR128RegClass, Op0);
3045 }
3046 return Register();
3047}
3048
3049Register fastEmit_ISD_FNEG_MVT_v2f64_r(MVT RetVT, Register Op0) {
3050 if (RetVT.SimpleTy != MVT::v2f64)
3051 return Register();
3052 if ((Subtarget->isNeonAvailable())) {
3053 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv2f64, RC: &AArch64::FPR128RegClass, Op0);
3054 }
3055 return Register();
3056}
3057
3058Register fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, Register Op0) {
3059 switch (VT.SimpleTy) {
3060 case MVT::bf16: return fastEmit_ISD_FNEG_MVT_bf16_r(RetVT, Op0);
3061 case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0);
3062 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
3063 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
3064 case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0);
3065 case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0);
3066 case MVT::v4bf16: return fastEmit_ISD_FNEG_MVT_v4bf16_r(RetVT, Op0);
3067 case MVT::v8bf16: return fastEmit_ISD_FNEG_MVT_v8bf16_r(RetVT, Op0);
3068 case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0);
3069 case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0);
3070 case MVT::v2f64: return fastEmit_ISD_FNEG_MVT_v2f64_r(RetVT, Op0);
3071 default: return Register();
3072 }
3073}
3074
3075// FastEmit functions for ISD::FP_EXTEND.
3076
3077Register fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Register Op0) {
3078 if ((Subtarget->hasFPARMv8())) {
3079 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSHr, RC: &AArch64::FPR32RegClass, Op0);
3080 }
3081 return Register();
3082}
3083
3084Register fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Register Op0) {
3085 if ((Subtarget->hasFPARMv8())) {
3086 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDHr, RC: &AArch64::FPR64RegClass, Op0);
3087 }
3088 return Register();
3089}
3090
3091Register fastEmit_ISD_FP_EXTEND_MVT_f16_r(MVT RetVT, Register Op0) {
3092switch (RetVT.SimpleTy) {
3093 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Op0);
3094 case MVT::f64: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Op0);
3095 default: return Register();
3096}
3097}
3098
3099Register fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
3100 if (RetVT.SimpleTy != MVT::f64)
3101 return Register();
3102 if ((Subtarget->hasFPARMv8())) {
3103 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDSr, RC: &AArch64::FPR64RegClass, Op0);
3104 }
3105 return Register();
3106}
3107
3108Register fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3109 if (RetVT.SimpleTy != MVT::v4f32)
3110 return Register();
3111 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv4i16, RC: &AArch64::FPR128RegClass, Op0);
3112}
3113
3114Register fastEmit_ISD_FP_EXTEND_MVT_v4bf16_r(MVT RetVT, Register Op0) {
3115 if (RetVT.SimpleTy != MVT::v4f32)
3116 return Register();
3117 if ((Subtarget->isNeonAvailable())) {
3118 return fastEmitInst_r(MachineInstOpcode: AArch64::SHLLv4i16, RC: &AArch64::FPR128RegClass, Op0);
3119 }
3120 return Register();
3121}
3122
3123Register fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3124 if (RetVT.SimpleTy != MVT::v2f64)
3125 return Register();
3126 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv2i32, RC: &AArch64::FPR128RegClass, Op0);
3127}
3128
3129Register fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3130 switch (VT.SimpleTy) {
3131 case MVT::f16: return fastEmit_ISD_FP_EXTEND_MVT_f16_r(RetVT, Op0);
3132 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
3133 case MVT::v4f16: return fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
3134 case MVT::v4bf16: return fastEmit_ISD_FP_EXTEND_MVT_v4bf16_r(RetVT, Op0);
3135 case MVT::v2f32: return fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(RetVT, Op0);
3136 default: return Register();
3137 }
3138}
3139
3140// FastEmit functions for ISD::FP_ROUND.
3141
3142Register fastEmit_ISD_FP_ROUND_MVT_f32_MVT_bf16_r(Register Op0) {
3143 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
3144 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVT, RC: &AArch64::FPR16RegClass, Op0);
3145 }
3146 return Register();
3147}
3148
3149Register fastEmit_ISD_FP_ROUND_MVT_f32_MVT_f16_r(Register Op0) {
3150 if ((Subtarget->hasFPARMv8())) {
3151 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHSr, RC: &AArch64::FPR16RegClass, Op0);
3152 }
3153 return Register();
3154}
3155
3156Register fastEmit_ISD_FP_ROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3157switch (RetVT.SimpleTy) {
3158 case MVT::bf16: return fastEmit_ISD_FP_ROUND_MVT_f32_MVT_bf16_r(Op0);
3159 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f32_MVT_f16_r(Op0);
3160 default: return Register();
3161}
3162}
3163
3164Register fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Register Op0) {
3165 if ((Subtarget->hasFPARMv8())) {
3166 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHDr, RC: &AArch64::FPR16RegClass, Op0);
3167 }
3168 return Register();
3169}
3170
3171Register fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Register Op0) {
3172 if ((Subtarget->hasFPARMv8())) {
3173 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSDr, RC: &AArch64::FPR32RegClass, Op0);
3174 }
3175 return Register();
3176}
3177
3178Register fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3179switch (RetVT.SimpleTy) {
3180 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Op0);
3181 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Op0);
3182 default: return Register();
3183}
3184}
3185
3186Register fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Register Op0) {
3187 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
3188}
3189
3190Register fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Register Op0) {
3191 if ((Subtarget->hasBF16()) && (Subtarget->isNeonAvailable())) {
3192 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVTN, RC: &AArch64::FPR64RegClass, Op0);
3193 }
3194 return Register();
3195}
3196
3197Register fastEmit_ISD_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3198switch (RetVT.SimpleTy) {
3199 case MVT::v4f16: return fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Op0);
3200 case MVT::v4bf16: return fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Op0);
3201 default: return Register();
3202}
3203}
3204
3205Register fastEmit_ISD_FP_ROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
3206 if (RetVT.SimpleTy != MVT::v2f32)
3207 return Register();
3208 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
3209}
3210
3211Register fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
3212 switch (VT.SimpleTy) {
3213 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f32_r(RetVT, Op0);
3214 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
3215 case MVT::v4f32: return fastEmit_ISD_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
3216 case MVT::v2f64: return fastEmit_ISD_FP_ROUND_MVT_v2f64_r(RetVT, Op0);
3217 default: return Register();
3218 }
3219}
3220
3221// FastEmit functions for ISD::FP_TO_SINT.
3222
3223Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
3224 if ((Subtarget->hasFPRCVT())) {
3225 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSHr, RC: &AArch64::FPR32RegClass, Op0);
3226 }
3227 if ((Subtarget->hasFullFP16())) {
3228 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
3229 }
3230 return Register();
3231}
3232
3233Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
3234 if ((Subtarget->hasFPRCVT())) {
3235 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDHr, RC: &AArch64::FPR64RegClass, Op0);
3236 }
3237 if ((Subtarget->hasFullFP16())) {
3238 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
3239 }
3240 return Register();
3241}
3242
3243Register fastEmit_ISD_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
3244switch (RetVT.SimpleTy) {
3245 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
3246 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
3247 default: return Register();
3248}
3249}
3250
3251Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
3252 if ((Subtarget->hasFPARMv8())) {
3253 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
3254 }
3255 return Register();
3256}
3257
3258Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
3259 if ((Subtarget->hasFPRCVT())) {
3260 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDSr, RC: &AArch64::FPR64RegClass, Op0);
3261 }
3262 if ((Subtarget->hasFPARMv8())) {
3263 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
3264 }
3265 return Register();
3266}
3267
3268Register fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
3269switch (RetVT.SimpleTy) {
3270 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
3271 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
3272 default: return Register();
3273}
3274}
3275
3276Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
3277 if ((Subtarget->hasFPRCVT())) {
3278 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSDr, RC: &AArch64::FPR32RegClass, Op0);
3279 }
3280 if ((Subtarget->hasFPARMv8())) {
3281 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
3282 }
3283 return Register();
3284}
3285
3286Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
3287 if ((Subtarget->hasFPARMv8())) {
3288 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
3289 }
3290 return Register();
3291}
3292
3293Register fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
3294switch (RetVT.SimpleTy) {
3295 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
3296 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
3297 default: return Register();
3298}
3299}
3300
3301Register fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3302 if (RetVT.SimpleTy != MVT::v4i16)
3303 return Register();
3304 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3305 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
3306 }
3307 return Register();
3308}
3309
3310Register fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3311 if (RetVT.SimpleTy != MVT::v8i16)
3312 return Register();
3313 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3314 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
3315 }
3316 return Register();
3317}
3318
3319Register fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3320 if (RetVT.SimpleTy != MVT::v2i32)
3321 return Register();
3322 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3323 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
3324 }
3325 return Register();
3326}
3327
3328Register fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3329 if (RetVT.SimpleTy != MVT::v4i32)
3330 return Register();
3331 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3332 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
3333 }
3334 return Register();
3335}
3336
3337Register fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3338 if (RetVT.SimpleTy != MVT::v2i64)
3339 return Register();
3340 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3341 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
3342 }
3343 return Register();
3344}
3345
3346Register fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
3347 switch (VT.SimpleTy) {
3348 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
3349 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
3350 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
3351 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
3352 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
3353 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
3354 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
3355 case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0);
3356 default: return Register();
3357 }
3358}
3359
3360// FastEmit functions for ISD::FP_TO_UINT.
3361
3362Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
3363 if ((Subtarget->hasFPRCVT())) {
3364 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSHr, RC: &AArch64::FPR32RegClass, Op0);
3365 }
3366 if ((Subtarget->hasFullFP16())) {
3367 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
3368 }
3369 return Register();
3370}
3371
3372Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
3373 if ((Subtarget->hasFPRCVT())) {
3374 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDHr, RC: &AArch64::FPR64RegClass, Op0);
3375 }
3376 if ((Subtarget->hasFullFP16())) {
3377 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
3378 }
3379 return Register();
3380}
3381
3382Register fastEmit_ISD_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
3383switch (RetVT.SimpleTy) {
3384 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
3385 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
3386 default: return Register();
3387}
3388}
3389
3390Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
3391 if ((Subtarget->hasFPARMv8())) {
3392 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
3393 }
3394 return Register();
3395}
3396
3397Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
3398 if ((Subtarget->hasFPRCVT())) {
3399 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDSr, RC: &AArch64::FPR64RegClass, Op0);
3400 }
3401 if ((Subtarget->hasFPARMv8())) {
3402 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
3403 }
3404 return Register();
3405}
3406
3407Register fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
3408switch (RetVT.SimpleTy) {
3409 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
3410 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
3411 default: return Register();
3412}
3413}
3414
3415Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
3416 if ((Subtarget->hasFPRCVT())) {
3417 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSDr, RC: &AArch64::FPR32RegClass, Op0);
3418 }
3419 if ((Subtarget->hasFPARMv8())) {
3420 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
3421 }
3422 return Register();
3423}
3424
3425Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
3426 if ((Subtarget->hasFPARMv8())) {
3427 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
3428 }
3429 return Register();
3430}
3431
3432Register fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
3433switch (RetVT.SimpleTy) {
3434 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
3435 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
3436 default: return Register();
3437}
3438}
3439
3440Register fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3441 if (RetVT.SimpleTy != MVT::v4i16)
3442 return Register();
3443 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3444 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
3445 }
3446 return Register();
3447}
3448
3449Register fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3450 if (RetVT.SimpleTy != MVT::v8i16)
3451 return Register();
3452 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3453 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
3454 }
3455 return Register();
3456}
3457
3458Register fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3459 if (RetVT.SimpleTy != MVT::v2i32)
3460 return Register();
3461 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3462 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
3463 }
3464 return Register();
3465}
3466
3467Register fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3468 if (RetVT.SimpleTy != MVT::v4i32)
3469 return Register();
3470 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3471 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
3472 }
3473 return Register();
3474}
3475
3476Register fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3477 if (RetVT.SimpleTy != MVT::v2i64)
3478 return Register();
3479 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3480 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
3481 }
3482 return Register();
3483}
3484
3485Register fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
3486 switch (VT.SimpleTy) {
3487 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
3488 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
3489 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
3490 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
3491 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
3492 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
3493 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
3494 case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0);
3495 default: return Register();
3496 }
3497}
3498
3499// FastEmit functions for ISD::FRINT.
3500
3501Register fastEmit_ISD_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
3502 if (RetVT.SimpleTy != MVT::f16)
3503 return Register();
3504 if ((Subtarget->hasFullFP16())) {
3505 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXHr, RC: &AArch64::FPR16RegClass, Op0);
3506 }
3507 return Register();
3508}
3509
3510Register fastEmit_ISD_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
3511 if (RetVT.SimpleTy != MVT::f32)
3512 return Register();
3513 if ((Subtarget->hasFPARMv8())) {
3514 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXSr, RC: &AArch64::FPR32RegClass, Op0);
3515 }
3516 return Register();
3517}
3518
3519Register fastEmit_ISD_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
3520 if (RetVT.SimpleTy != MVT::f64)
3521 return Register();
3522 if ((Subtarget->hasFPARMv8())) {
3523 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXDr, RC: &AArch64::FPR64RegClass, Op0);
3524 }
3525 return Register();
3526}
3527
3528Register fastEmit_ISD_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3529 if (RetVT.SimpleTy != MVT::v4f16)
3530 return Register();
3531 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3532 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f16, RC: &AArch64::FPR64RegClass, Op0);
3533 }
3534 return Register();
3535}
3536
3537Register fastEmit_ISD_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3538 if (RetVT.SimpleTy != MVT::v8f16)
3539 return Register();
3540 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3541 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv8f16, RC: &AArch64::FPR128RegClass, Op0);
3542 }
3543 return Register();
3544}
3545
3546Register fastEmit_ISD_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3547 if (RetVT.SimpleTy != MVT::v2f32)
3548 return Register();
3549 if ((Subtarget->isNeonAvailable())) {
3550 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f32, RC: &AArch64::FPR64RegClass, Op0);
3551 }
3552 return Register();
3553}
3554
3555Register fastEmit_ISD_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3556 if (RetVT.SimpleTy != MVT::v4f32)
3557 return Register();
3558 if ((Subtarget->isNeonAvailable())) {
3559 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f32, RC: &AArch64::FPR128RegClass, Op0);
3560 }
3561 return Register();
3562}
3563
3564Register fastEmit_ISD_FRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3565 if (RetVT.SimpleTy != MVT::v2f64)
3566 return Register();
3567 if ((Subtarget->isNeonAvailable())) {
3568 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f64, RC: &AArch64::FPR128RegClass, Op0);
3569 }
3570 return Register();
3571}
3572
3573Register fastEmit_ISD_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
3574 switch (VT.SimpleTy) {
3575 case MVT::f16: return fastEmit_ISD_FRINT_MVT_f16_r(RetVT, Op0);
3576 case MVT::f32: return fastEmit_ISD_FRINT_MVT_f32_r(RetVT, Op0);
3577 case MVT::f64: return fastEmit_ISD_FRINT_MVT_f64_r(RetVT, Op0);
3578 case MVT::v4f16: return fastEmit_ISD_FRINT_MVT_v4f16_r(RetVT, Op0);
3579 case MVT::v8f16: return fastEmit_ISD_FRINT_MVT_v8f16_r(RetVT, Op0);
3580 case MVT::v2f32: return fastEmit_ISD_FRINT_MVT_v2f32_r(RetVT, Op0);
3581 case MVT::v4f32: return fastEmit_ISD_FRINT_MVT_v4f32_r(RetVT, Op0);
3582 case MVT::v2f64: return fastEmit_ISD_FRINT_MVT_v2f64_r(RetVT, Op0);
3583 default: return Register();
3584 }
3585}
3586
3587// FastEmit functions for ISD::FROUND.
3588
3589Register fastEmit_ISD_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
3590 if (RetVT.SimpleTy != MVT::f16)
3591 return Register();
3592 if ((Subtarget->hasFullFP16())) {
3593 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAHr, RC: &AArch64::FPR16RegClass, Op0);
3594 }
3595 return Register();
3596}
3597
3598Register fastEmit_ISD_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3599 if (RetVT.SimpleTy != MVT::f32)
3600 return Register();
3601 if ((Subtarget->hasFPARMv8())) {
3602 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTASr, RC: &AArch64::FPR32RegClass, Op0);
3603 }
3604 return Register();
3605}
3606
3607Register fastEmit_ISD_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3608 if (RetVT.SimpleTy != MVT::f64)
3609 return Register();
3610 if ((Subtarget->hasFPARMv8())) {
3611 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTADr, RC: &AArch64::FPR64RegClass, Op0);
3612 }
3613 return Register();
3614}
3615
3616Register fastEmit_ISD_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3617 if (RetVT.SimpleTy != MVT::v4f16)
3618 return Register();
3619 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3620 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f16, RC: &AArch64::FPR64RegClass, Op0);
3621 }
3622 return Register();
3623}
3624
3625Register fastEmit_ISD_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
3626 if (RetVT.SimpleTy != MVT::v8f16)
3627 return Register();
3628 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3629 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv8f16, RC: &AArch64::FPR128RegClass, Op0);
3630 }
3631 return Register();
3632}
3633
3634Register fastEmit_ISD_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3635 if (RetVT.SimpleTy != MVT::v2f32)
3636 return Register();
3637 if ((Subtarget->isNeonAvailable())) {
3638 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f32, RC: &AArch64::FPR64RegClass, Op0);
3639 }
3640 return Register();
3641}
3642
3643Register fastEmit_ISD_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3644 if (RetVT.SimpleTy != MVT::v4f32)
3645 return Register();
3646 if ((Subtarget->isNeonAvailable())) {
3647 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f32, RC: &AArch64::FPR128RegClass, Op0);
3648 }
3649 return Register();
3650}
3651
3652Register fastEmit_ISD_FROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
3653 if (RetVT.SimpleTy != MVT::v2f64)
3654 return Register();
3655 if ((Subtarget->isNeonAvailable())) {
3656 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f64, RC: &AArch64::FPR128RegClass, Op0);
3657 }
3658 return Register();
3659}
3660
3661Register fastEmit_ISD_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
3662 switch (VT.SimpleTy) {
3663 case MVT::f16: return fastEmit_ISD_FROUND_MVT_f16_r(RetVT, Op0);
3664 case MVT::f32: return fastEmit_ISD_FROUND_MVT_f32_r(RetVT, Op0);
3665 case MVT::f64: return fastEmit_ISD_FROUND_MVT_f64_r(RetVT, Op0);
3666 case MVT::v4f16: return fastEmit_ISD_FROUND_MVT_v4f16_r(RetVT, Op0);
3667 case MVT::v8f16: return fastEmit_ISD_FROUND_MVT_v8f16_r(RetVT, Op0);
3668 case MVT::v2f32: return fastEmit_ISD_FROUND_MVT_v2f32_r(RetVT, Op0);
3669 case MVT::v4f32: return fastEmit_ISD_FROUND_MVT_v4f32_r(RetVT, Op0);
3670 case MVT::v2f64: return fastEmit_ISD_FROUND_MVT_v2f64_r(RetVT, Op0);
3671 default: return Register();
3672 }
3673}
3674
3675// FastEmit functions for ISD::FROUNDEVEN.
3676
3677Register fastEmit_ISD_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
3678 if (RetVT.SimpleTy != MVT::f16)
3679 return Register();
3680 if ((Subtarget->hasFullFP16())) {
3681 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNHr, RC: &AArch64::FPR16RegClass, Op0);
3682 }
3683 return Register();
3684}
3685
3686Register fastEmit_ISD_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
3687 if (RetVT.SimpleTy != MVT::f32)
3688 return Register();
3689 if ((Subtarget->hasFPARMv8())) {
3690 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNSr, RC: &AArch64::FPR32RegClass, Op0);
3691 }
3692 return Register();
3693}
3694
3695Register fastEmit_ISD_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
3696 if (RetVT.SimpleTy != MVT::f64)
3697 return Register();
3698 if ((Subtarget->hasFPARMv8())) {
3699 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNDr, RC: &AArch64::FPR64RegClass, Op0);
3700 }
3701 return Register();
3702}
3703
3704Register fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
3705 if (RetVT.SimpleTy != MVT::v4f16)
3706 return Register();
3707 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3708 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f16, RC: &AArch64::FPR64RegClass, Op0);
3709 }
3710 return Register();
3711}
3712
3713Register fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
3714 if (RetVT.SimpleTy != MVT::v8f16)
3715 return Register();
3716 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3717 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv8f16, RC: &AArch64::FPR128RegClass, Op0);
3718 }
3719 return Register();
3720}
3721
3722Register fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
3723 if (RetVT.SimpleTy != MVT::v2f32)
3724 return Register();
3725 if ((Subtarget->isNeonAvailable())) {
3726 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f32, RC: &AArch64::FPR64RegClass, Op0);
3727 }
3728 return Register();
3729}
3730
3731Register fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
3732 if (RetVT.SimpleTy != MVT::v4f32)
3733 return Register();
3734 if ((Subtarget->isNeonAvailable())) {
3735 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f32, RC: &AArch64::FPR128RegClass, Op0);
3736 }
3737 return Register();
3738}
3739
3740Register fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, Register Op0) {
3741 if (RetVT.SimpleTy != MVT::v2f64)
3742 return Register();
3743 if ((Subtarget->isNeonAvailable())) {
3744 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f64, RC: &AArch64::FPR128RegClass, Op0);
3745 }
3746 return Register();
3747}
3748
3749Register fastEmit_ISD_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
3750 switch (VT.SimpleTy) {
3751 case MVT::f16: return fastEmit_ISD_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
3752 case MVT::f32: return fastEmit_ISD_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
3753 case MVT::f64: return fastEmit_ISD_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
3754 case MVT::v4f16: return fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
3755 case MVT::v8f16: return fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
3756 case MVT::v2f32: return fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
3757 case MVT::v4f32: return fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
3758 case MVT::v2f64: return fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0);
3759 default: return Register();
3760 }
3761}
3762
3763// FastEmit functions for ISD::FSQRT.
3764
3765Register fastEmit_ISD_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
3766 if (RetVT.SimpleTy != MVT::f16)
3767 return Register();
3768 if ((Subtarget->hasFullFP16())) {
3769 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTHr, RC: &AArch64::FPR16RegClass, Op0);
3770 }
3771 return Register();
3772}
3773
3774Register fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
3775 if (RetVT.SimpleTy != MVT::f32)
3776 return Register();
3777 if ((Subtarget->hasFPARMv8())) {
3778 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTSr, RC: &AArch64::FPR32RegClass, Op0);
3779 }
3780 return Register();
3781}
3782
3783Register fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
3784 if (RetVT.SimpleTy != MVT::f64)
3785 return Register();
3786 if ((Subtarget->hasFPARMv8())) {
3787 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTDr, RC: &AArch64::FPR64RegClass, Op0);
3788 }
3789 return Register();
3790}
3791
3792Register fastEmit_ISD_FSQRT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3793 if (RetVT.SimpleTy != MVT::v4f16)
3794 return Register();
3795 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3796 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f16, RC: &AArch64::FPR64RegClass, Op0);
3797 }
3798 return Register();
3799}
3800
3801Register fastEmit_ISD_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3802 if (RetVT.SimpleTy != MVT::v8f16)
3803 return Register();
3804 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3805 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv8f16, RC: &AArch64::FPR128RegClass, Op0);
3806 }
3807 return Register();
3808}
3809
3810Register fastEmit_ISD_FSQRT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3811 if (RetVT.SimpleTy != MVT::v2f32)
3812 return Register();
3813 if ((Subtarget->isNeonAvailable())) {
3814 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f32, RC: &AArch64::FPR64RegClass, Op0);
3815 }
3816 return Register();
3817}
3818
3819Register fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3820 if (RetVT.SimpleTy != MVT::v4f32)
3821 return Register();
3822 if ((Subtarget->isNeonAvailable())) {
3823 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f32, RC: &AArch64::FPR128RegClass, Op0);
3824 }
3825 return Register();
3826}
3827
3828Register fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3829 if (RetVT.SimpleTy != MVT::v2f64)
3830 return Register();
3831 if ((Subtarget->isNeonAvailable())) {
3832 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f64, RC: &AArch64::FPR128RegClass, Op0);
3833 }
3834 return Register();
3835}
3836
3837Register fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
3838 switch (VT.SimpleTy) {
3839 case MVT::f16: return fastEmit_ISD_FSQRT_MVT_f16_r(RetVT, Op0);
3840 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
3841 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
3842 case MVT::v4f16: return fastEmit_ISD_FSQRT_MVT_v4f16_r(RetVT, Op0);
3843 case MVT::v8f16: return fastEmit_ISD_FSQRT_MVT_v8f16_r(RetVT, Op0);
3844 case MVT::v2f32: return fastEmit_ISD_FSQRT_MVT_v2f32_r(RetVT, Op0);
3845 case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0);
3846 case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0);
3847 default: return Register();
3848 }
3849}
3850
3851// FastEmit functions for ISD::FTRUNC.
3852
3853Register fastEmit_ISD_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
3854 if (RetVT.SimpleTy != MVT::f16)
3855 return Register();
3856 if ((Subtarget->hasFullFP16())) {
3857 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZHr, RC: &AArch64::FPR16RegClass, Op0);
3858 }
3859 return Register();
3860}
3861
3862Register fastEmit_ISD_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
3863 if (RetVT.SimpleTy != MVT::f32)
3864 return Register();
3865 if ((Subtarget->hasFPARMv8())) {
3866 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZSr, RC: &AArch64::FPR32RegClass, Op0);
3867 }
3868 return Register();
3869}
3870
3871Register fastEmit_ISD_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
3872 if (RetVT.SimpleTy != MVT::f64)
3873 return Register();
3874 if ((Subtarget->hasFPARMv8())) {
3875 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZDr, RC: &AArch64::FPR64RegClass, Op0);
3876 }
3877 return Register();
3878}
3879
3880Register fastEmit_ISD_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
3881 if (RetVT.SimpleTy != MVT::v4f16)
3882 return Register();
3883 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3884 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f16, RC: &AArch64::FPR64RegClass, Op0);
3885 }
3886 return Register();
3887}
3888
3889Register fastEmit_ISD_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
3890 if (RetVT.SimpleTy != MVT::v8f16)
3891 return Register();
3892 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3893 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv8f16, RC: &AArch64::FPR128RegClass, Op0);
3894 }
3895 return Register();
3896}
3897
3898Register fastEmit_ISD_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
3899 if (RetVT.SimpleTy != MVT::v2f32)
3900 return Register();
3901 if ((Subtarget->isNeonAvailable())) {
3902 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f32, RC: &AArch64::FPR64RegClass, Op0);
3903 }
3904 return Register();
3905}
3906
3907Register fastEmit_ISD_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
3908 if (RetVT.SimpleTy != MVT::v4f32)
3909 return Register();
3910 if ((Subtarget->isNeonAvailable())) {
3911 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f32, RC: &AArch64::FPR128RegClass, Op0);
3912 }
3913 return Register();
3914}
3915
3916Register fastEmit_ISD_FTRUNC_MVT_v2f64_r(MVT RetVT, Register Op0) {
3917 if (RetVT.SimpleTy != MVT::v2f64)
3918 return Register();
3919 if ((Subtarget->isNeonAvailable())) {
3920 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f64, RC: &AArch64::FPR128RegClass, Op0);
3921 }
3922 return Register();
3923}
3924
3925Register fastEmit_ISD_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
3926 switch (VT.SimpleTy) {
3927 case MVT::f16: return fastEmit_ISD_FTRUNC_MVT_f16_r(RetVT, Op0);
3928 case MVT::f32: return fastEmit_ISD_FTRUNC_MVT_f32_r(RetVT, Op0);
3929 case MVT::f64: return fastEmit_ISD_FTRUNC_MVT_f64_r(RetVT, Op0);
3930 case MVT::v4f16: return fastEmit_ISD_FTRUNC_MVT_v4f16_r(RetVT, Op0);
3931 case MVT::v8f16: return fastEmit_ISD_FTRUNC_MVT_v8f16_r(RetVT, Op0);
3932 case MVT::v2f32: return fastEmit_ISD_FTRUNC_MVT_v2f32_r(RetVT, Op0);
3933 case MVT::v4f32: return fastEmit_ISD_FTRUNC_MVT_v4f32_r(RetVT, Op0);
3934 case MVT::v2f64: return fastEmit_ISD_FTRUNC_MVT_v2f64_r(RetVT, Op0);
3935 default: return Register();
3936 }
3937}
3938
3939// FastEmit functions for ISD::LLROUND.
3940
3941Register fastEmit_ISD_LLROUND_MVT_f16_r(MVT RetVT, Register Op0) {
3942 if (RetVT.SimpleTy != MVT::i64)
3943 return Register();
3944 if ((Subtarget->hasFPRCVT())) {
3945 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDHr, RC: &AArch64::FPR64RegClass, Op0);
3946 }
3947 if ((Subtarget->hasFullFP16())) {
3948 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
3949 }
3950 return Register();
3951}
3952
3953Register fastEmit_ISD_LLROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3954 if (RetVT.SimpleTy != MVT::i64)
3955 return Register();
3956 if ((Subtarget->hasFPRCVT())) {
3957 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDSr, RC: &AArch64::FPR64RegClass, Op0);
3958 }
3959 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
3960}
3961
3962Register fastEmit_ISD_LLROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3963 if (RetVT.SimpleTy != MVT::i64)
3964 return Register();
3965 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
3966}
3967
3968Register fastEmit_ISD_LLROUND_r(MVT VT, MVT RetVT, Register Op0) {
3969 switch (VT.SimpleTy) {
3970 case MVT::f16: return fastEmit_ISD_LLROUND_MVT_f16_r(RetVT, Op0);
3971 case MVT::f32: return fastEmit_ISD_LLROUND_MVT_f32_r(RetVT, Op0);
3972 case MVT::f64: return fastEmit_ISD_LLROUND_MVT_f64_r(RetVT, Op0);
3973 default: return Register();
3974 }
3975}
3976
3977// FastEmit functions for ISD::LROUND.
3978
3979Register fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(Register Op0) {
3980 if ((Subtarget->hasFPRCVT())) {
3981 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASSHr, RC: &AArch64::FPR32RegClass, Op0);
3982 }
3983 if ((Subtarget->hasFullFP16())) {
3984 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWHr, RC: &AArch64::GPR32RegClass, Op0);
3985 }
3986 return Register();
3987}
3988
3989Register fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(Register Op0) {
3990 if ((Subtarget->hasFPRCVT())) {
3991 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDHr, RC: &AArch64::FPR64RegClass, Op0);
3992 }
3993 if ((Subtarget->hasFullFP16())) {
3994 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
3995 }
3996 return Register();
3997}
3998
3999Register fastEmit_ISD_LROUND_MVT_f16_r(MVT RetVT, Register Op0) {
4000switch (RetVT.SimpleTy) {
4001 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(Op0);
4002 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(Op0);
4003 default: return Register();
4004}
4005}
4006
4007Register fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(Register Op0) {
4008 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWSr, RC: &AArch64::GPR32RegClass, Op0);
4009}
4010
4011Register fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(Register Op0) {
4012 if ((Subtarget->hasFPRCVT())) {
4013 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDSr, RC: &AArch64::FPR64RegClass, Op0);
4014 }
4015 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
4016}
4017
4018Register fastEmit_ISD_LROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4019switch (RetVT.SimpleTy) {
4020 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(Op0);
4021 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(Op0);
4022 default: return Register();
4023}
4024}
4025
4026Register fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(Register Op0) {
4027 if ((Subtarget->hasFPRCVT())) {
4028 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASSDr, RC: &AArch64::FPR32RegClass, Op0);
4029 }
4030 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWDr, RC: &AArch64::GPR32RegClass, Op0);
4031}
4032
4033Register fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(Register Op0) {
4034 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
4035}
4036
4037Register fastEmit_ISD_LROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4038switch (RetVT.SimpleTy) {
4039 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(Op0);
4040 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(Op0);
4041 default: return Register();
4042}
4043}
4044
4045Register fastEmit_ISD_LROUND_r(MVT VT, MVT RetVT, Register Op0) {
4046 switch (VT.SimpleTy) {
4047 case MVT::f16: return fastEmit_ISD_LROUND_MVT_f16_r(RetVT, Op0);
4048 case MVT::f32: return fastEmit_ISD_LROUND_MVT_f32_r(RetVT, Op0);
4049 case MVT::f64: return fastEmit_ISD_LROUND_MVT_f64_r(RetVT, Op0);
4050 default: return Register();
4051 }
4052}
4053
4054// FastEmit functions for ISD::SINT_TO_FP.
4055
4056Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
4057 if ((Subtarget->hasFPRCVT())) {
4058 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
4059 }
4060 if ((Subtarget->hasFullFP16())) {
4061 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
4062 }
4063 return Register();
4064}
4065
4066Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
4067 if ((Subtarget->hasFPARMv8())) {
4068 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
4069 }
4070 return Register();
4071}
4072
4073Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
4074 if ((Subtarget->hasFPRCVT())) {
4075 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
4076 }
4077 if ((Subtarget->hasFPARMv8())) {
4078 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
4079 }
4080 return Register();
4081}
4082
4083Register fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
4084switch (RetVT.SimpleTy) {
4085 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
4086 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
4087 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
4088 default: return Register();
4089}
4090}
4091
4092Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
4093 if ((Subtarget->hasFPRCVT())) {
4094 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
4095 }
4096 if ((Subtarget->hasFullFP16())) {
4097 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
4098 }
4099 return Register();
4100}
4101
4102Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
4103 if ((Subtarget->hasFPRCVT())) {
4104 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
4105 }
4106 if ((Subtarget->hasFPARMv8())) {
4107 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
4108 }
4109 return Register();
4110}
4111
4112Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
4113 if ((Subtarget->hasFPARMv8())) {
4114 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
4115 }
4116 return Register();
4117}
4118
4119Register fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
4120switch (RetVT.SimpleTy) {
4121 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
4122 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
4123 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
4124 default: return Register();
4125}
4126}
4127
4128Register fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
4129 if (RetVT.SimpleTy != MVT::v4f16)
4130 return Register();
4131 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4132 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
4133 }
4134 return Register();
4135}
4136
4137Register fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
4138 if (RetVT.SimpleTy != MVT::v8f16)
4139 return Register();
4140 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4141 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
4142 }
4143 return Register();
4144}
4145
4146Register fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
4147 if (RetVT.SimpleTy != MVT::v2f32)
4148 return Register();
4149 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4150 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
4151 }
4152 return Register();
4153}
4154
4155Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
4156 if (RetVT.SimpleTy != MVT::v4f32)
4157 return Register();
4158 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4159 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
4160 }
4161 return Register();
4162}
4163
4164Register fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
4165 if (RetVT.SimpleTy != MVT::v2f64)
4166 return Register();
4167 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4168 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
4169 }
4170 return Register();
4171}
4172
4173Register fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
4174 switch (VT.SimpleTy) {
4175 case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
4176 case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
4177 case MVT::v4i16: return fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
4178 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
4179 case MVT::v2i32: return fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
4180 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
4181 case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
4182 default: return Register();
4183 }
4184}
4185
4186// FastEmit functions for ISD::SPLAT_VECTOR.
4187
4188Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(Register Op0) {
4189 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4190 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_B, RC: &AArch64::ZPRRegClass, Op0);
4191 }
4192 return Register();
4193}
4194
4195Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(Register Op0) {
4196 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4197 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_H, RC: &AArch64::ZPRRegClass, Op0);
4198 }
4199 return Register();
4200}
4201
4202Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(Register Op0) {
4203 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4204 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_S, RC: &AArch64::ZPRRegClass, Op0);
4205 }
4206 return Register();
4207}
4208
4209Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(MVT RetVT, Register Op0) {
4210switch (RetVT.SimpleTy) {
4211 case MVT::nxv16i8: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(Op0);
4212 case MVT::nxv8i16: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(Op0);
4213 case MVT::nxv4i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(Op0);
4214 default: return Register();
4215}
4216}
4217
4218Register fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(MVT RetVT, Register Op0) {
4219 if (RetVT.SimpleTy != MVT::nxv2i64)
4220 return Register();
4221 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4222 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_D, RC: &AArch64::ZPRRegClass, Op0);
4223 }
4224 return Register();
4225}
4226
4227Register fastEmit_ISD_SPLAT_VECTOR_r(MVT VT, MVT RetVT, Register Op0) {
4228 switch (VT.SimpleTy) {
4229 case MVT::i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(RetVT, Op0);
4230 case MVT::i64: return fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(RetVT, Op0);
4231 default: return Register();
4232 }
4233}
4234
4235// FastEmit functions for ISD::STRICT_FCEIL.
4236
4237Register fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
4238 if (RetVT.SimpleTy != MVT::f16)
4239 return Register();
4240 if ((Subtarget->hasFullFP16())) {
4241 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPHr, RC: &AArch64::FPR16RegClass, Op0);
4242 }
4243 return Register();
4244}
4245
4246Register fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
4247 if (RetVT.SimpleTy != MVT::f32)
4248 return Register();
4249 if ((Subtarget->hasFPARMv8())) {
4250 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPSr, RC: &AArch64::FPR32RegClass, Op0);
4251 }
4252 return Register();
4253}
4254
4255Register fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
4256 if (RetVT.SimpleTy != MVT::f64)
4257 return Register();
4258 if ((Subtarget->hasFPARMv8())) {
4259 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPDr, RC: &AArch64::FPR64RegClass, Op0);
4260 }
4261 return Register();
4262}
4263
4264Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
4265 if (RetVT.SimpleTy != MVT::v4f16)
4266 return Register();
4267 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4268 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f16, RC: &AArch64::FPR64RegClass, Op0);
4269 }
4270 return Register();
4271}
4272
4273Register fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
4274 if (RetVT.SimpleTy != MVT::v8f16)
4275 return Register();
4276 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4277 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv8f16, RC: &AArch64::FPR128RegClass, Op0);
4278 }
4279 return Register();
4280}
4281
4282Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
4283 if (RetVT.SimpleTy != MVT::v2f32)
4284 return Register();
4285 if ((Subtarget->isNeonAvailable())) {
4286 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f32, RC: &AArch64::FPR64RegClass, Op0);
4287 }
4288 return Register();
4289}
4290
4291Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
4292 if (RetVT.SimpleTy != MVT::v4f32)
4293 return Register();
4294 if ((Subtarget->isNeonAvailable())) {
4295 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f32, RC: &AArch64::FPR128RegClass, Op0);
4296 }
4297 return Register();
4298}
4299
4300Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(MVT RetVT, Register Op0) {
4301 if (RetVT.SimpleTy != MVT::v2f64)
4302 return Register();
4303 if ((Subtarget->isNeonAvailable())) {
4304 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f64, RC: &AArch64::FPR128RegClass, Op0);
4305 }
4306 return Register();
4307}
4308
4309Register fastEmit_ISD_STRICT_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
4310 switch (VT.SimpleTy) {
4311 case MVT::f16: return fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(RetVT, Op0);
4312 case MVT::f32: return fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(RetVT, Op0);
4313 case MVT::f64: return fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(RetVT, Op0);
4314 case MVT::v4f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(RetVT, Op0);
4315 case MVT::v8f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(RetVT, Op0);
4316 case MVT::v2f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(RetVT, Op0);
4317 case MVT::v4f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(RetVT, Op0);
4318 case MVT::v2f64: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(RetVT, Op0);
4319 default: return Register();
4320 }
4321}
4322
4323// FastEmit functions for ISD::STRICT_FFLOOR.
4324
4325Register fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
4326 if (RetVT.SimpleTy != MVT::f16)
4327 return Register();
4328 if ((Subtarget->hasFullFP16())) {
4329 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMHr, RC: &AArch64::FPR16RegClass, Op0);
4330 }
4331 return Register();
4332}
4333
4334Register fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
4335 if (RetVT.SimpleTy != MVT::f32)
4336 return Register();
4337 if ((Subtarget->hasFPARMv8())) {
4338 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMSr, RC: &AArch64::FPR32RegClass, Op0);
4339 }
4340 return Register();
4341}
4342
4343Register fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
4344 if (RetVT.SimpleTy != MVT::f64)
4345 return Register();
4346 if ((Subtarget->hasFPARMv8())) {
4347 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMDr, RC: &AArch64::FPR64RegClass, Op0);
4348 }
4349 return Register();
4350}
4351
4352Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
4353 if (RetVT.SimpleTy != MVT::v4f16)
4354 return Register();
4355 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4356 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f16, RC: &AArch64::FPR64RegClass, Op0);
4357 }
4358 return Register();
4359}
4360
4361Register fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
4362 if (RetVT.SimpleTy != MVT::v8f16)
4363 return Register();
4364 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4365 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv8f16, RC: &AArch64::FPR128RegClass, Op0);
4366 }
4367 return Register();
4368}
4369
4370Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
4371 if (RetVT.SimpleTy != MVT::v2f32)
4372 return Register();
4373 if ((Subtarget->isNeonAvailable())) {
4374 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f32, RC: &AArch64::FPR64RegClass, Op0);
4375 }
4376 return Register();
4377}
4378
4379Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
4380 if (RetVT.SimpleTy != MVT::v4f32)
4381 return Register();
4382 if ((Subtarget->isNeonAvailable())) {
4383 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f32, RC: &AArch64::FPR128RegClass, Op0);
4384 }
4385 return Register();
4386}
4387
4388Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(MVT RetVT, Register Op0) {
4389 if (RetVT.SimpleTy != MVT::v2f64)
4390 return Register();
4391 if ((Subtarget->isNeonAvailable())) {
4392 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f64, RC: &AArch64::FPR128RegClass, Op0);
4393 }
4394 return Register();
4395}
4396
4397Register fastEmit_ISD_STRICT_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
4398 switch (VT.SimpleTy) {
4399 case MVT::f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(RetVT, Op0);
4400 case MVT::f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(RetVT, Op0);
4401 case MVT::f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(RetVT, Op0);
4402 case MVT::v4f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(RetVT, Op0);
4403 case MVT::v8f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(RetVT, Op0);
4404 case MVT::v2f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(RetVT, Op0);
4405 case MVT::v4f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(RetVT, Op0);
4406 case MVT::v2f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(RetVT, Op0);
4407 default: return Register();
4408 }
4409}
4410
4411// FastEmit functions for ISD::STRICT_FNEARBYINT.
4412
4413Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
4414 if (RetVT.SimpleTy != MVT::f16)
4415 return Register();
4416 if ((Subtarget->hasFullFP16())) {
4417 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIHr, RC: &AArch64::FPR16RegClass, Op0);
4418 }
4419 return Register();
4420}
4421
4422Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
4423 if (RetVT.SimpleTy != MVT::f32)
4424 return Register();
4425 if ((Subtarget->hasFPARMv8())) {
4426 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTISr, RC: &AArch64::FPR32RegClass, Op0);
4427 }
4428 return Register();
4429}
4430
4431Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
4432 if (RetVT.SimpleTy != MVT::f64)
4433 return Register();
4434 if ((Subtarget->hasFPARMv8())) {
4435 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIDr, RC: &AArch64::FPR64RegClass, Op0);
4436 }
4437 return Register();
4438}
4439
4440Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4441 if (RetVT.SimpleTy != MVT::v4f16)
4442 return Register();
4443 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4444 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f16, RC: &AArch64::FPR64RegClass, Op0);
4445 }
4446 return Register();
4447}
4448
4449Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4450 if (RetVT.SimpleTy != MVT::v8f16)
4451 return Register();
4452 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4453 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv8f16, RC: &AArch64::FPR128RegClass, Op0);
4454 }
4455 return Register();
4456}
4457
4458Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4459 if (RetVT.SimpleTy != MVT::v2f32)
4460 return Register();
4461 if ((Subtarget->isNeonAvailable())) {
4462 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f32, RC: &AArch64::FPR64RegClass, Op0);
4463 }
4464 return Register();
4465}
4466
4467Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4468 if (RetVT.SimpleTy != MVT::v4f32)
4469 return Register();
4470 if ((Subtarget->isNeonAvailable())) {
4471 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f32, RC: &AArch64::FPR128RegClass, Op0);
4472 }
4473 return Register();
4474}
4475
4476Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4477 if (RetVT.SimpleTy != MVT::v2f64)
4478 return Register();
4479 if ((Subtarget->isNeonAvailable())) {
4480 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f64, RC: &AArch64::FPR128RegClass, Op0);
4481 }
4482 return Register();
4483}
4484
4485Register fastEmit_ISD_STRICT_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
4486 switch (VT.SimpleTy) {
4487 case MVT::f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(RetVT, Op0);
4488 case MVT::f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(RetVT, Op0);
4489 case MVT::f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(RetVT, Op0);
4490 case MVT::v4f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(RetVT, Op0);
4491 case MVT::v8f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(RetVT, Op0);
4492 case MVT::v2f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(RetVT, Op0);
4493 case MVT::v4f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(RetVT, Op0);
4494 case MVT::v2f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(RetVT, Op0);
4495 default: return Register();
4496 }
4497}
4498
4499// FastEmit functions for ISD::STRICT_FP_EXTEND.
4500
4501Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(Register Op0) {
4502 if ((Subtarget->hasFPARMv8())) {
4503 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSHr, RC: &AArch64::FPR32RegClass, Op0);
4504 }
4505 return Register();
4506}
4507
4508Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(Register Op0) {
4509 if ((Subtarget->hasFPARMv8())) {
4510 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDHr, RC: &AArch64::FPR64RegClass, Op0);
4511 }
4512 return Register();
4513}
4514
4515Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(MVT RetVT, Register Op0) {
4516switch (RetVT.SimpleTy) {
4517 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(Op0);
4518 case MVT::f64: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(Op0);
4519 default: return Register();
4520}
4521}
4522
4523Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
4524 if (RetVT.SimpleTy != MVT::f64)
4525 return Register();
4526 if ((Subtarget->hasFPARMv8())) {
4527 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDSr, RC: &AArch64::FPR64RegClass, Op0);
4528 }
4529 return Register();
4530}
4531
4532Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
4533 if (RetVT.SimpleTy != MVT::v4f32)
4534 return Register();
4535 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv4i16, RC: &AArch64::FPR128RegClass, Op0);
4536}
4537
4538Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4bf16_r(MVT RetVT, Register Op0) {
4539 if (RetVT.SimpleTy != MVT::v4f32)
4540 return Register();
4541 if ((Subtarget->isNeonAvailable())) {
4542 return fastEmitInst_r(MachineInstOpcode: AArch64::SHLLv4i16, RC: &AArch64::FPR128RegClass, Op0);
4543 }
4544 return Register();
4545}
4546
4547Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(MVT RetVT, Register Op0) {
4548 if (RetVT.SimpleTy != MVT::v2f64)
4549 return Register();
4550 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv2i32, RC: &AArch64::FPR128RegClass, Op0);
4551}
4552
4553Register fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
4554 switch (VT.SimpleTy) {
4555 case MVT::f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(RetVT, Op0);
4556 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0);
4557 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
4558 case MVT::v4bf16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4bf16_r(RetVT, Op0);
4559 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(RetVT, Op0);
4560 default: return Register();
4561 }
4562}
4563
4564// FastEmit functions for ISD::STRICT_FP_ROUND.
4565
4566Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_bf16_r(Register Op0) {
4567 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
4568 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVT, RC: &AArch64::FPR16RegClass, Op0);
4569 }
4570 return Register();
4571}
4572
4573Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_f16_r(Register Op0) {
4574 if ((Subtarget->hasFPARMv8())) {
4575 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHSr, RC: &AArch64::FPR16RegClass, Op0);
4576 }
4577 return Register();
4578}
4579
4580Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4581switch (RetVT.SimpleTy) {
4582 case MVT::bf16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_bf16_r(Op0);
4583 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_f16_r(Op0);
4584 default: return Register();
4585}
4586}
4587
4588Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(Register Op0) {
4589 if ((Subtarget->hasFPARMv8())) {
4590 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHDr, RC: &AArch64::FPR16RegClass, Op0);
4591 }
4592 return Register();
4593}
4594
4595Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(Register Op0) {
4596 if ((Subtarget->hasFPARMv8())) {
4597 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSDr, RC: &AArch64::FPR32RegClass, Op0);
4598 }
4599 return Register();
4600}
4601
4602Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4603switch (RetVT.SimpleTy) {
4604 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(Op0);
4605 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(Op0);
4606 default: return Register();
4607}
4608}
4609
4610Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Register Op0) {
4611 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
4612}
4613
4614Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Register Op0) {
4615 if ((Subtarget->hasBF16()) && (Subtarget->isNeonAvailable())) {
4616 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVTN, RC: &AArch64::FPR64RegClass, Op0);
4617 }
4618 return Register();
4619}
4620
4621Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
4622switch (RetVT.SimpleTy) {
4623 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Op0);
4624 case MVT::v4bf16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Op0);
4625 default: return Register();
4626}
4627}
4628
4629Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
4630 if (RetVT.SimpleTy != MVT::v2f32)
4631 return Register();
4632 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
4633}
4634
4635Register fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
4636 switch (VT.SimpleTy) {
4637 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(RetVT, Op0);
4638 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0);
4639 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
4640 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(RetVT, Op0);
4641 default: return Register();
4642 }
4643}
4644
4645// FastEmit functions for ISD::STRICT_FP_TO_SINT.
4646
4647Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
4648 if ((Subtarget->hasFPRCVT())) {
4649 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSHr, RC: &AArch64::FPR32RegClass, Op0);
4650 }
4651 if ((Subtarget->hasFullFP16())) {
4652 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
4653 }
4654 return Register();
4655}
4656
4657Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
4658 if ((Subtarget->hasFPRCVT())) {
4659 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDHr, RC: &AArch64::FPR64RegClass, Op0);
4660 }
4661 if ((Subtarget->hasFullFP16())) {
4662 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
4663 }
4664 return Register();
4665}
4666
4667Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
4668switch (RetVT.SimpleTy) {
4669 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
4670 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
4671 default: return Register();
4672}
4673}
4674
4675Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
4676 if ((Subtarget->hasFPARMv8())) {
4677 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
4678 }
4679 return Register();
4680}
4681
4682Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
4683 if ((Subtarget->hasFPRCVT())) {
4684 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDSr, RC: &AArch64::FPR64RegClass, Op0);
4685 }
4686 if ((Subtarget->hasFPARMv8())) {
4687 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
4688 }
4689 return Register();
4690}
4691
4692Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
4693switch (RetVT.SimpleTy) {
4694 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
4695 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
4696 default: return Register();
4697}
4698}
4699
4700Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
4701 if ((Subtarget->hasFPRCVT())) {
4702 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSDr, RC: &AArch64::FPR32RegClass, Op0);
4703 }
4704 if ((Subtarget->hasFPARMv8())) {
4705 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
4706 }
4707 return Register();
4708}
4709
4710Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
4711 if ((Subtarget->hasFPARMv8())) {
4712 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
4713 }
4714 return Register();
4715}
4716
4717Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
4718switch (RetVT.SimpleTy) {
4719 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
4720 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
4721 default: return Register();
4722}
4723}
4724
4725Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4726 if (RetVT.SimpleTy != MVT::v4i16)
4727 return Register();
4728 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4729 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
4730 }
4731 return Register();
4732}
4733
4734Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4735 if (RetVT.SimpleTy != MVT::v8i16)
4736 return Register();
4737 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4738 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
4739 }
4740 return Register();
4741}
4742
4743Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4744 if (RetVT.SimpleTy != MVT::v2i32)
4745 return Register();
4746 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4747 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
4748 }
4749 return Register();
4750}
4751
4752Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4753 if (RetVT.SimpleTy != MVT::v4i32)
4754 return Register();
4755 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4756 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
4757 }
4758 return Register();
4759}
4760
4761Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4762 if (RetVT.SimpleTy != MVT::v2i64)
4763 return Register();
4764 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4765 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
4766 }
4767 return Register();
4768}
4769
4770Register fastEmit_ISD_STRICT_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
4771 switch (VT.SimpleTy) {
4772 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
4773 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
4774 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
4775 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
4776 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
4777 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
4778 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
4779 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0);
4780 default: return Register();
4781 }
4782}
4783
4784// FastEmit functions for ISD::STRICT_FP_TO_UINT.
4785
4786Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
4787 if ((Subtarget->hasFPRCVT())) {
4788 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSHr, RC: &AArch64::FPR32RegClass, Op0);
4789 }
4790 if ((Subtarget->hasFullFP16())) {
4791 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
4792 }
4793 return Register();
4794}
4795
4796Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
4797 if ((Subtarget->hasFPRCVT())) {
4798 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDHr, RC: &AArch64::FPR64RegClass, Op0);
4799 }
4800 if ((Subtarget->hasFullFP16())) {
4801 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
4802 }
4803 return Register();
4804}
4805
4806Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
4807switch (RetVT.SimpleTy) {
4808 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
4809 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
4810 default: return Register();
4811}
4812}
4813
4814Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
4815 if ((Subtarget->hasFPARMv8())) {
4816 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
4817 }
4818 return Register();
4819}
4820
4821Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
4822 if ((Subtarget->hasFPRCVT())) {
4823 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDSr, RC: &AArch64::FPR64RegClass, Op0);
4824 }
4825 if ((Subtarget->hasFPARMv8())) {
4826 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
4827 }
4828 return Register();
4829}
4830
4831Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
4832switch (RetVT.SimpleTy) {
4833 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
4834 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
4835 default: return Register();
4836}
4837}
4838
4839Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
4840 if ((Subtarget->hasFPRCVT())) {
4841 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSDr, RC: &AArch64::FPR32RegClass, Op0);
4842 }
4843 if ((Subtarget->hasFPARMv8())) {
4844 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
4845 }
4846 return Register();
4847}
4848
4849Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
4850 if ((Subtarget->hasFPARMv8())) {
4851 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
4852 }
4853 return Register();
4854}
4855
4856Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
4857switch (RetVT.SimpleTy) {
4858 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
4859 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
4860 default: return Register();
4861}
4862}
4863
4864Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4865 if (RetVT.SimpleTy != MVT::v4i16)
4866 return Register();
4867 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4868 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
4869 }
4870 return Register();
4871}
4872
4873Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4874 if (RetVT.SimpleTy != MVT::v8i16)
4875 return Register();
4876 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4877 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
4878 }
4879 return Register();
4880}
4881
4882Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4883 if (RetVT.SimpleTy != MVT::v2i32)
4884 return Register();
4885 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4886 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
4887 }
4888 return Register();
4889}
4890
4891Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4892 if (RetVT.SimpleTy != MVT::v4i32)
4893 return Register();
4894 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4895 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
4896 }
4897 return Register();
4898}
4899
4900Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4901 if (RetVT.SimpleTy != MVT::v2i64)
4902 return Register();
4903 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4904 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
4905 }
4906 return Register();
4907}
4908
4909Register fastEmit_ISD_STRICT_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
4910 switch (VT.SimpleTy) {
4911 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
4912 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
4913 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
4914 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
4915 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
4916 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
4917 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
4918 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0);
4919 default: return Register();
4920 }
4921}
4922
4923// FastEmit functions for ISD::STRICT_FRINT.
4924
4925Register fastEmit_ISD_STRICT_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
4926 if (RetVT.SimpleTy != MVT::f16)
4927 return Register();
4928 if ((Subtarget->hasFullFP16())) {
4929 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXHr, RC: &AArch64::FPR16RegClass, Op0);
4930 }
4931 return Register();
4932}
4933
4934Register fastEmit_ISD_STRICT_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
4935 if (RetVT.SimpleTy != MVT::f32)
4936 return Register();
4937 if ((Subtarget->hasFPARMv8())) {
4938 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXSr, RC: &AArch64::FPR32RegClass, Op0);
4939 }
4940 return Register();
4941}
4942
4943Register fastEmit_ISD_STRICT_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
4944 if (RetVT.SimpleTy != MVT::f64)
4945 return Register();
4946 if ((Subtarget->hasFPARMv8())) {
4947 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXDr, RC: &AArch64::FPR64RegClass, Op0);
4948 }
4949 return Register();
4950}
4951
4952Register fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4953 if (RetVT.SimpleTy != MVT::v4f16)
4954 return Register();
4955 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4956 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f16, RC: &AArch64::FPR64RegClass, Op0);
4957 }
4958 return Register();
4959}
4960
4961Register fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4962 if (RetVT.SimpleTy != MVT::v8f16)
4963 return Register();
4964 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4965 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv8f16, RC: &AArch64::FPR128RegClass, Op0);
4966 }
4967 return Register();
4968}
4969
4970Register fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4971 if (RetVT.SimpleTy != MVT::v2f32)
4972 return Register();
4973 if ((Subtarget->isNeonAvailable())) {
4974 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f32, RC: &AArch64::FPR64RegClass, Op0);
4975 }
4976 return Register();
4977}
4978
4979Register fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4980 if (RetVT.SimpleTy != MVT::v4f32)
4981 return Register();
4982 if ((Subtarget->isNeonAvailable())) {
4983 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f32, RC: &AArch64::FPR128RegClass, Op0);
4984 }
4985 return Register();
4986}
4987
4988Register fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4989 if (RetVT.SimpleTy != MVT::v2f64)
4990 return Register();
4991 if ((Subtarget->isNeonAvailable())) {
4992 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f64, RC: &AArch64::FPR128RegClass, Op0);
4993 }
4994 return Register();
4995}
4996
4997Register fastEmit_ISD_STRICT_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
4998 switch (VT.SimpleTy) {
4999 case MVT::f16: return fastEmit_ISD_STRICT_FRINT_MVT_f16_r(RetVT, Op0);
5000 case MVT::f32: return fastEmit_ISD_STRICT_FRINT_MVT_f32_r(RetVT, Op0);
5001 case MVT::f64: return fastEmit_ISD_STRICT_FRINT_MVT_f64_r(RetVT, Op0);
5002 case MVT::v4f16: return fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(RetVT, Op0);
5003 case MVT::v8f16: return fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(RetVT, Op0);
5004 case MVT::v2f32: return fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(RetVT, Op0);
5005 case MVT::v4f32: return fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(RetVT, Op0);
5006 case MVT::v2f64: return fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(RetVT, Op0);
5007 default: return Register();
5008 }
5009}
5010
5011// FastEmit functions for ISD::STRICT_FROUND.
5012
5013Register fastEmit_ISD_STRICT_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5014 if (RetVT.SimpleTy != MVT::f16)
5015 return Register();
5016 if ((Subtarget->hasFullFP16())) {
5017 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAHr, RC: &AArch64::FPR16RegClass, Op0);
5018 }
5019 return Register();
5020}
5021
5022Register fastEmit_ISD_STRICT_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5023 if (RetVT.SimpleTy != MVT::f32)
5024 return Register();
5025 if ((Subtarget->hasFPARMv8())) {
5026 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTASr, RC: &AArch64::FPR32RegClass, Op0);
5027 }
5028 return Register();
5029}
5030
5031Register fastEmit_ISD_STRICT_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5032 if (RetVT.SimpleTy != MVT::f64)
5033 return Register();
5034 if ((Subtarget->hasFPARMv8())) {
5035 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTADr, RC: &AArch64::FPR64RegClass, Op0);
5036 }
5037 return Register();
5038}
5039
5040Register fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
5041 if (RetVT.SimpleTy != MVT::v4f16)
5042 return Register();
5043 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5044 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f16, RC: &AArch64::FPR64RegClass, Op0);
5045 }
5046 return Register();
5047}
5048
5049Register fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
5050 if (RetVT.SimpleTy != MVT::v8f16)
5051 return Register();
5052 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5053 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv8f16, RC: &AArch64::FPR128RegClass, Op0);
5054 }
5055 return Register();
5056}
5057
5058Register fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
5059 if (RetVT.SimpleTy != MVT::v2f32)
5060 return Register();
5061 if ((Subtarget->isNeonAvailable())) {
5062 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f32, RC: &AArch64::FPR64RegClass, Op0);
5063 }
5064 return Register();
5065}
5066
5067Register fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
5068 if (RetVT.SimpleTy != MVT::v4f32)
5069 return Register();
5070 if ((Subtarget->isNeonAvailable())) {
5071 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f32, RC: &AArch64::FPR128RegClass, Op0);
5072 }
5073 return Register();
5074}
5075
5076Register fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
5077 if (RetVT.SimpleTy != MVT::v2f64)
5078 return Register();
5079 if ((Subtarget->isNeonAvailable())) {
5080 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f64, RC: &AArch64::FPR128RegClass, Op0);
5081 }
5082 return Register();
5083}
5084
5085Register fastEmit_ISD_STRICT_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
5086 switch (VT.SimpleTy) {
5087 case MVT::f16: return fastEmit_ISD_STRICT_FROUND_MVT_f16_r(RetVT, Op0);
5088 case MVT::f32: return fastEmit_ISD_STRICT_FROUND_MVT_f32_r(RetVT, Op0);
5089 case MVT::f64: return fastEmit_ISD_STRICT_FROUND_MVT_f64_r(RetVT, Op0);
5090 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(RetVT, Op0);
5091 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(RetVT, Op0);
5092 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(RetVT, Op0);
5093 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(RetVT, Op0);
5094 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(RetVT, Op0);
5095 default: return Register();
5096 }
5097}
5098
5099// FastEmit functions for ISD::STRICT_FROUNDEVEN.
5100
5101Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
5102 if (RetVT.SimpleTy != MVT::f16)
5103 return Register();
5104 if ((Subtarget->hasFullFP16())) {
5105 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNHr, RC: &AArch64::FPR16RegClass, Op0);
5106 }
5107 return Register();
5108}
5109
5110Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
5111 if (RetVT.SimpleTy != MVT::f32)
5112 return Register();
5113 if ((Subtarget->hasFPARMv8())) {
5114 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNSr, RC: &AArch64::FPR32RegClass, Op0);
5115 }
5116 return Register();
5117}
5118
5119Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
5120 if (RetVT.SimpleTy != MVT::f64)
5121 return Register();
5122 if ((Subtarget->hasFPARMv8())) {
5123 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNDr, RC: &AArch64::FPR64RegClass, Op0);
5124 }
5125 return Register();
5126}
5127
5128Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
5129 if (RetVT.SimpleTy != MVT::v4f16)
5130 return Register();
5131 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5132 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f16, RC: &AArch64::FPR64RegClass, Op0);
5133 }
5134 return Register();
5135}
5136
5137Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
5138 if (RetVT.SimpleTy != MVT::v8f16)
5139 return Register();
5140 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5141 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv8f16, RC: &AArch64::FPR128RegClass, Op0);
5142 }
5143 return Register();
5144}
5145
5146Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
5147 if (RetVT.SimpleTy != MVT::v2f32)
5148 return Register();
5149 if ((Subtarget->isNeonAvailable())) {
5150 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f32, RC: &AArch64::FPR64RegClass, Op0);
5151 }
5152 return Register();
5153}
5154
5155Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
5156 if (RetVT.SimpleTy != MVT::v4f32)
5157 return Register();
5158 if ((Subtarget->isNeonAvailable())) {
5159 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f32, RC: &AArch64::FPR128RegClass, Op0);
5160 }
5161 return Register();
5162}
5163
5164Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, Register Op0) {
5165 if (RetVT.SimpleTy != MVT::v2f64)
5166 return Register();
5167 if ((Subtarget->isNeonAvailable())) {
5168 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f64, RC: &AArch64::FPR128RegClass, Op0);
5169 }
5170 return Register();
5171}
5172
5173Register fastEmit_ISD_STRICT_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
5174 switch (VT.SimpleTy) {
5175 case MVT::f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
5176 case MVT::f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
5177 case MVT::f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
5178 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
5179 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
5180 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
5181 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
5182 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0);
5183 default: return Register();
5184 }
5185}
5186
5187// FastEmit functions for ISD::STRICT_FSQRT.
5188
5189Register fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
5190 if (RetVT.SimpleTy != MVT::f16)
5191 return Register();
5192 if ((Subtarget->hasFullFP16())) {
5193 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTHr, RC: &AArch64::FPR16RegClass, Op0);
5194 }
5195 return Register();
5196}
5197
5198Register fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
5199 if (RetVT.SimpleTy != MVT::f32)
5200 return Register();
5201 if ((Subtarget->hasFPARMv8())) {
5202 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTSr, RC: &AArch64::FPR32RegClass, Op0);
5203 }
5204 return Register();
5205}
5206
5207Register fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
5208 if (RetVT.SimpleTy != MVT::f64)
5209 return Register();
5210 if ((Subtarget->hasFPARMv8())) {
5211 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTDr, RC: &AArch64::FPR64RegClass, Op0);
5212 }
5213 return Register();
5214}
5215
5216Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(MVT RetVT, Register Op0) {
5217 if (RetVT.SimpleTy != MVT::v4f16)
5218 return Register();
5219 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5220 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f16, RC: &AArch64::FPR64RegClass, Op0);
5221 }
5222 return Register();
5223}
5224
5225Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
5226 if (RetVT.SimpleTy != MVT::v8f16)
5227 return Register();
5228 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5229 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv8f16, RC: &AArch64::FPR128RegClass, Op0);
5230 }
5231 return Register();
5232}
5233
5234Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(MVT RetVT, Register Op0) {
5235 if (RetVT.SimpleTy != MVT::v2f32)
5236 return Register();
5237 if ((Subtarget->isNeonAvailable())) {
5238 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f32, RC: &AArch64::FPR64RegClass, Op0);
5239 }
5240 return Register();
5241}
5242
5243Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
5244 if (RetVT.SimpleTy != MVT::v4f32)
5245 return Register();
5246 if ((Subtarget->isNeonAvailable())) {
5247 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f32, RC: &AArch64::FPR128RegClass, Op0);
5248 }
5249 return Register();
5250}
5251
5252Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
5253 if (RetVT.SimpleTy != MVT::v2f64)
5254 return Register();
5255 if ((Subtarget->isNeonAvailable())) {
5256 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f64, RC: &AArch64::FPR128RegClass, Op0);
5257 }
5258 return Register();
5259}
5260
5261Register fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
5262 switch (VT.SimpleTy) {
5263 case MVT::f16: return fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(RetVT, Op0);
5264 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0);
5265 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0);
5266 case MVT::v4f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(RetVT, Op0);
5267 case MVT::v8f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(RetVT, Op0);
5268 case MVT::v2f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(RetVT, Op0);
5269 case MVT::v4f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(RetVT, Op0);
5270 case MVT::v2f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(RetVT, Op0);
5271 default: return Register();
5272 }
5273}
5274
5275// FastEmit functions for ISD::STRICT_FTRUNC.
5276
5277Register fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
5278 if (RetVT.SimpleTy != MVT::f16)
5279 return Register();
5280 if ((Subtarget->hasFullFP16())) {
5281 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZHr, RC: &AArch64::FPR16RegClass, Op0);
5282 }
5283 return Register();
5284}
5285
5286Register fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
5287 if (RetVT.SimpleTy != MVT::f32)
5288 return Register();
5289 if ((Subtarget->hasFPARMv8())) {
5290 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZSr, RC: &AArch64::FPR32RegClass, Op0);
5291 }
5292 return Register();
5293}
5294
5295Register fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
5296 if (RetVT.SimpleTy != MVT::f64)
5297 return Register();
5298 if ((Subtarget->hasFPARMv8())) {
5299 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZDr, RC: &AArch64::FPR64RegClass, Op0);
5300 }
5301 return Register();
5302}
5303
5304Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
5305 if (RetVT.SimpleTy != MVT::v4f16)
5306 return Register();
5307 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5308 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f16, RC: &AArch64::FPR64RegClass, Op0);
5309 }
5310 return Register();
5311}
5312
5313Register fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
5314 if (RetVT.SimpleTy != MVT::v8f16)
5315 return Register();
5316 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5317 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv8f16, RC: &AArch64::FPR128RegClass, Op0);
5318 }
5319 return Register();
5320}
5321
5322Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
5323 if (RetVT.SimpleTy != MVT::v2f32)
5324 return Register();
5325 if ((Subtarget->isNeonAvailable())) {
5326 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f32, RC: &AArch64::FPR64RegClass, Op0);
5327 }
5328 return Register();
5329}
5330
5331Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
5332 if (RetVT.SimpleTy != MVT::v4f32)
5333 return Register();
5334 if ((Subtarget->isNeonAvailable())) {
5335 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f32, RC: &AArch64::FPR128RegClass, Op0);
5336 }
5337 return Register();
5338}
5339
5340Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(MVT RetVT, Register Op0) {
5341 if (RetVT.SimpleTy != MVT::v2f64)
5342 return Register();
5343 if ((Subtarget->isNeonAvailable())) {
5344 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f64, RC: &AArch64::FPR128RegClass, Op0);
5345 }
5346 return Register();
5347}
5348
5349Register fastEmit_ISD_STRICT_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
5350 switch (VT.SimpleTy) {
5351 case MVT::f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(RetVT, Op0);
5352 case MVT::f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(RetVT, Op0);
5353 case MVT::f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(RetVT, Op0);
5354 case MVT::v4f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(RetVT, Op0);
5355 case MVT::v8f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(RetVT, Op0);
5356 case MVT::v2f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(RetVT, Op0);
5357 case MVT::v4f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(RetVT, Op0);
5358 case MVT::v2f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(RetVT, Op0);
5359 default: return Register();
5360 }
5361}
5362
5363// FastEmit functions for ISD::STRICT_LLROUND.
5364
5365Register fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5366 if (RetVT.SimpleTy != MVT::i64)
5367 return Register();
5368 if ((Subtarget->hasFullFP16())) {
5369 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
5370 }
5371 return Register();
5372}
5373
5374Register fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5375 if (RetVT.SimpleTy != MVT::i64)
5376 return Register();
5377 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
5378}
5379
5380Register fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5381 if (RetVT.SimpleTy != MVT::i64)
5382 return Register();
5383 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
5384}
5385
5386Register fastEmit_ISD_STRICT_LLROUND_r(MVT VT, MVT RetVT, Register Op0) {
5387 switch (VT.SimpleTy) {
5388 case MVT::f16: return fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(RetVT, Op0);
5389 case MVT::f32: return fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(RetVT, Op0);
5390 case MVT::f64: return fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(RetVT, Op0);
5391 default: return Register();
5392 }
5393}
5394
5395// FastEmit functions for ISD::STRICT_LROUND.
5396
5397Register fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(Register Op0) {
5398 if ((Subtarget->hasFullFP16())) {
5399 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWHr, RC: &AArch64::GPR32RegClass, Op0);
5400 }
5401 return Register();
5402}
5403
5404Register fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(Register Op0) {
5405 if ((Subtarget->hasFullFP16())) {
5406 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
5407 }
5408 return Register();
5409}
5410
5411Register fastEmit_ISD_STRICT_LROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5412switch (RetVT.SimpleTy) {
5413 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(Op0);
5414 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(Op0);
5415 default: return Register();
5416}
5417}
5418
5419Register fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(Register Op0) {
5420 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWSr, RC: &AArch64::GPR32RegClass, Op0);
5421}
5422
5423Register fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(Register Op0) {
5424 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
5425}
5426
5427Register fastEmit_ISD_STRICT_LROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5428switch (RetVT.SimpleTy) {
5429 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(Op0);
5430 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(Op0);
5431 default: return Register();
5432}
5433}
5434
5435Register fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(Register Op0) {
5436 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWDr, RC: &AArch64::GPR32RegClass, Op0);
5437}
5438
5439Register fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(Register Op0) {
5440 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
5441}
5442
5443Register fastEmit_ISD_STRICT_LROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5444switch (RetVT.SimpleTy) {
5445 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(Op0);
5446 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(Op0);
5447 default: return Register();
5448}
5449}
5450
5451Register fastEmit_ISD_STRICT_LROUND_r(MVT VT, MVT RetVT, Register Op0) {
5452 switch (VT.SimpleTy) {
5453 case MVT::f16: return fastEmit_ISD_STRICT_LROUND_MVT_f16_r(RetVT, Op0);
5454 case MVT::f32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_r(RetVT, Op0);
5455 case MVT::f64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_r(RetVT, Op0);
5456 default: return Register();
5457 }
5458}
5459
5460// FastEmit functions for ISD::STRICT_SINT_TO_FP.
5461
5462Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5463 if ((Subtarget->hasFPRCVT())) {
5464 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5465 }
5466 if ((Subtarget->hasFullFP16())) {
5467 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5468 }
5469 return Register();
5470}
5471
5472Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5473 if ((Subtarget->hasFPARMv8())) {
5474 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5475 }
5476 return Register();
5477}
5478
5479Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5480 if ((Subtarget->hasFPRCVT())) {
5481 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5482 }
5483 if ((Subtarget->hasFPARMv8())) {
5484 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5485 }
5486 return Register();
5487}
5488
5489Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5490switch (RetVT.SimpleTy) {
5491 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5492 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5493 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5494 default: return Register();
5495}
5496}
5497
5498Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5499 if ((Subtarget->hasFPRCVT())) {
5500 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5501 }
5502 if ((Subtarget->hasFullFP16())) {
5503 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5504 }
5505 return Register();
5506}
5507
5508Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5509 if ((Subtarget->hasFPRCVT())) {
5510 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5511 }
5512 if ((Subtarget->hasFPARMv8())) {
5513 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5514 }
5515 return Register();
5516}
5517
5518Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5519 if ((Subtarget->hasFPARMv8())) {
5520 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5521 }
5522 return Register();
5523}
5524
5525Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5526switch (RetVT.SimpleTy) {
5527 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5528 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5529 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5530 default: return Register();
5531}
5532}
5533
5534Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5535 if (RetVT.SimpleTy != MVT::v4f16)
5536 return Register();
5537 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5538 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5539 }
5540 return Register();
5541}
5542
5543Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5544 if (RetVT.SimpleTy != MVT::v8f16)
5545 return Register();
5546 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5547 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5548 }
5549 return Register();
5550}
5551
5552Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5553 if (RetVT.SimpleTy != MVT::v2f32)
5554 return Register();
5555 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5556 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5557 }
5558 return Register();
5559}
5560
5561Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5562 if (RetVT.SimpleTy != MVT::v4f32)
5563 return Register();
5564 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5565 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5566 }
5567 return Register();
5568}
5569
5570Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5571 if (RetVT.SimpleTy != MVT::v2f64)
5572 return Register();
5573 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5574 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5575 }
5576 return Register();
5577}
5578
5579Register fastEmit_ISD_STRICT_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
5580 switch (VT.SimpleTy) {
5581 case MVT::i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
5582 case MVT::i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
5583 case MVT::v4i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
5584 case MVT::v8i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
5585 case MVT::v2i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
5586 case MVT::v4i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
5587 case MVT::v2i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
5588 default: return Register();
5589 }
5590}
5591
5592// FastEmit functions for ISD::STRICT_UINT_TO_FP.
5593
5594Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5595 if ((Subtarget->hasFPRCVT())) {
5596 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5597 }
5598 if ((Subtarget->hasFullFP16())) {
5599 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5600 }
5601 return Register();
5602}
5603
5604Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5605 if ((Subtarget->hasFPARMv8())) {
5606 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5607 }
5608 return Register();
5609}
5610
5611Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5612 if ((Subtarget->hasFPRCVT())) {
5613 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5614 }
5615 if ((Subtarget->hasFPARMv8())) {
5616 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5617 }
5618 return Register();
5619}
5620
5621Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5622switch (RetVT.SimpleTy) {
5623 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5624 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5625 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5626 default: return Register();
5627}
5628}
5629
5630Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5631 if ((Subtarget->hasFPRCVT())) {
5632 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5633 }
5634 if ((Subtarget->hasFullFP16())) {
5635 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5636 }
5637 return Register();
5638}
5639
5640Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5641 if ((Subtarget->hasFPRCVT())) {
5642 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5643 }
5644 if ((Subtarget->hasFPARMv8())) {
5645 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5646 }
5647 return Register();
5648}
5649
5650Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5651 if ((Subtarget->hasFPARMv8())) {
5652 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5653 }
5654 return Register();
5655}
5656
5657Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5658switch (RetVT.SimpleTy) {
5659 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5660 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5661 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5662 default: return Register();
5663}
5664}
5665
5666Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5667 if (RetVT.SimpleTy != MVT::v4f16)
5668 return Register();
5669 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5670 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5671 }
5672 return Register();
5673}
5674
5675Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5676 if (RetVT.SimpleTy != MVT::v8f16)
5677 return Register();
5678 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5679 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5680 }
5681 return Register();
5682}
5683
5684Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5685 if (RetVT.SimpleTy != MVT::v2f32)
5686 return Register();
5687 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5688 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5689 }
5690 return Register();
5691}
5692
5693Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5694 if (RetVT.SimpleTy != MVT::v4f32)
5695 return Register();
5696 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5697 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5698 }
5699 return Register();
5700}
5701
5702Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5703 if (RetVT.SimpleTy != MVT::v2f64)
5704 return Register();
5705 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5706 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5707 }
5708 return Register();
5709}
5710
5711Register fastEmit_ISD_STRICT_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
5712 switch (VT.SimpleTy) {
5713 case MVT::i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(RetVT, Op0);
5714 case MVT::i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(RetVT, Op0);
5715 case MVT::v4i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
5716 case MVT::v8i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
5717 case MVT::v2i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
5718 case MVT::v4i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
5719 case MVT::v2i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
5720 default: return Register();
5721 }
5722}
5723
5724// FastEmit functions for ISD::TRUNCATE.
5725
5726Register fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, Register Op0) {
5727 if (RetVT.SimpleTy != MVT::i32)
5728 return Register();
5729 return fastEmitInst_extractsubreg(RetVT, Op0, Idx: AArch64::sub_32);
5730}
5731
5732Register fastEmit_ISD_TRUNCATE_MVT_v8i16_r(MVT RetVT, Register Op0) {
5733 if (RetVT.SimpleTy != MVT::v8i8)
5734 return Register();
5735 if ((Subtarget->isNeonAvailable())) {
5736 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5737 }
5738 return Register();
5739}
5740
5741Register fastEmit_ISD_TRUNCATE_MVT_v4i32_r(MVT RetVT, Register Op0) {
5742 if (RetVT.SimpleTy != MVT::v4i16)
5743 return Register();
5744 if ((Subtarget->isNeonAvailable())) {
5745 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5746 }
5747 return Register();
5748}
5749
5750Register fastEmit_ISD_TRUNCATE_MVT_v2i64_r(MVT RetVT, Register Op0) {
5751 if (RetVT.SimpleTy != MVT::v2i32)
5752 return Register();
5753 if ((Subtarget->isNeonAvailable())) {
5754 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5755 }
5756 return Register();
5757}
5758
5759Register fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, Register Op0) {
5760 switch (VT.SimpleTy) {
5761 case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0);
5762 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i16_r(RetVT, Op0);
5763 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_MVT_v4i32_r(RetVT, Op0);
5764 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_MVT_v2i64_r(RetVT, Op0);
5765 default: return Register();
5766 }
5767}
5768
5769// FastEmit functions for ISD::TRUNCATE_SSAT_S.
5770
5771Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v8i16_r(MVT RetVT, Register Op0) {
5772 if (RetVT.SimpleTy != MVT::v8i8)
5773 return Register();
5774 if ((Subtarget->isNeonAvailable())) {
5775 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5776 }
5777 return Register();
5778}
5779
5780Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v4i32_r(MVT RetVT, Register Op0) {
5781 if (RetVT.SimpleTy != MVT::v4i16)
5782 return Register();
5783 if ((Subtarget->isNeonAvailable())) {
5784 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5785 }
5786 return Register();
5787}
5788
5789Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v2i64_r(MVT RetVT, Register Op0) {
5790 if (RetVT.SimpleTy != MVT::v2i32)
5791 return Register();
5792 if ((Subtarget->isNeonAvailable())) {
5793 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5794 }
5795 return Register();
5796}
5797
5798Register fastEmit_ISD_TRUNCATE_SSAT_S_r(MVT VT, MVT RetVT, Register Op0) {
5799 switch (VT.SimpleTy) {
5800 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v8i16_r(RetVT, Op0);
5801 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v4i32_r(RetVT, Op0);
5802 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v2i64_r(RetVT, Op0);
5803 default: return Register();
5804 }
5805}
5806
5807// FastEmit functions for ISD::TRUNCATE_SSAT_U.
5808
5809Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v8i16_r(MVT RetVT, Register Op0) {
5810 if (RetVT.SimpleTy != MVT::v8i8)
5811 return Register();
5812 if ((Subtarget->isNeonAvailable())) {
5813 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5814 }
5815 return Register();
5816}
5817
5818Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v4i32_r(MVT RetVT, Register Op0) {
5819 if (RetVT.SimpleTy != MVT::v4i16)
5820 return Register();
5821 if ((Subtarget->isNeonAvailable())) {
5822 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5823 }
5824 return Register();
5825}
5826
5827Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v2i64_r(MVT RetVT, Register Op0) {
5828 if (RetVT.SimpleTy != MVT::v2i32)
5829 return Register();
5830 if ((Subtarget->isNeonAvailable())) {
5831 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5832 }
5833 return Register();
5834}
5835
5836Register fastEmit_ISD_TRUNCATE_SSAT_U_r(MVT VT, MVT RetVT, Register Op0) {
5837 switch (VT.SimpleTy) {
5838 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v8i16_r(RetVT, Op0);
5839 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v4i32_r(RetVT, Op0);
5840 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v2i64_r(RetVT, Op0);
5841 default: return Register();
5842 }
5843}
5844
5845// FastEmit functions for ISD::TRUNCATE_USAT_U.
5846
5847Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v8i16_r(MVT RetVT, Register Op0) {
5848 if (RetVT.SimpleTy != MVT::v8i8)
5849 return Register();
5850 if ((Subtarget->isNeonAvailable())) {
5851 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5852 }
5853 return Register();
5854}
5855
5856Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v4i32_r(MVT RetVT, Register Op0) {
5857 if (RetVT.SimpleTy != MVT::v4i16)
5858 return Register();
5859 if ((Subtarget->isNeonAvailable())) {
5860 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5861 }
5862 return Register();
5863}
5864
5865Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v2i64_r(MVT RetVT, Register Op0) {
5866 if (RetVT.SimpleTy != MVT::v2i32)
5867 return Register();
5868 if ((Subtarget->isNeonAvailable())) {
5869 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5870 }
5871 return Register();
5872}
5873
5874Register fastEmit_ISD_TRUNCATE_USAT_U_r(MVT VT, MVT RetVT, Register Op0) {
5875 switch (VT.SimpleTy) {
5876 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v8i16_r(RetVT, Op0);
5877 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v4i32_r(RetVT, Op0);
5878 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v2i64_r(RetVT, Op0);
5879 default: return Register();
5880 }
5881}
5882
5883// FastEmit functions for ISD::UINT_TO_FP.
5884
5885Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5886 if ((Subtarget->hasFPRCVT())) {
5887 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5888 }
5889 if ((Subtarget->hasFullFP16())) {
5890 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5891 }
5892 return Register();
5893}
5894
5895Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5896 if ((Subtarget->hasFPARMv8())) {
5897 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5898 }
5899 return Register();
5900}
5901
5902Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5903 if ((Subtarget->hasFPRCVT())) {
5904 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5905 }
5906 if ((Subtarget->hasFPARMv8())) {
5907 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5908 }
5909 return Register();
5910}
5911
5912Register fastEmit_ISD_UINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5913switch (RetVT.SimpleTy) {
5914 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5915 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5916 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5917 default: return Register();
5918}
5919}
5920
5921Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5922 if ((Subtarget->hasFPRCVT())) {
5923 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5924 }
5925 if ((Subtarget->hasFullFP16())) {
5926 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5927 }
5928 return Register();
5929}
5930
5931Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5932 if ((Subtarget->hasFPRCVT())) {
5933 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5934 }
5935 if ((Subtarget->hasFPARMv8())) {
5936 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5937 }
5938 return Register();
5939}
5940
5941Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5942 if ((Subtarget->hasFPARMv8())) {
5943 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5944 }
5945 return Register();
5946}
5947
5948Register fastEmit_ISD_UINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5949switch (RetVT.SimpleTy) {
5950 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5951 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5952 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5953 default: return Register();
5954}
5955}
5956
5957Register fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5958 if (RetVT.SimpleTy != MVT::v4f16)
5959 return Register();
5960 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5961 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5962 }
5963 return Register();
5964}
5965
5966Register fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5967 if (RetVT.SimpleTy != MVT::v8f16)
5968 return Register();
5969 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5970 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5971 }
5972 return Register();
5973}
5974
5975Register fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5976 if (RetVT.SimpleTy != MVT::v2f32)
5977 return Register();
5978 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5979 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5980 }
5981 return Register();
5982}
5983
5984Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5985 if (RetVT.SimpleTy != MVT::v4f32)
5986 return Register();
5987 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5988 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5989 }
5990 return Register();
5991}
5992
5993Register fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5994 if (RetVT.SimpleTy != MVT::v2f64)
5995 return Register();
5996 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5997 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5998 }
5999 return Register();
6000}
6001
6002Register fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
6003 switch (VT.SimpleTy) {
6004 case MVT::i32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_r(RetVT, Op0);
6005 case MVT::i64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_r(RetVT, Op0);
6006 case MVT::v4i16: return fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
6007 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
6008 case MVT::v2i32: return fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
6009 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
6010 case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
6011 default: return Register();
6012 }
6013}
6014
6015// FastEmit functions for ISD::VECREDUCE_ADD.
6016
6017Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i8_r(MVT RetVT, Register Op0) {
6018 if (RetVT.SimpleTy != MVT::i8)
6019 return Register();
6020 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6021}
6022
6023Register fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(MVT RetVT, Register Op0) {
6024 if (RetVT.SimpleTy != MVT::i8)
6025 return Register();
6026 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6027}
6028
6029Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i16_r(MVT RetVT, Register Op0) {
6030 if (RetVT.SimpleTy != MVT::i16)
6031 return Register();
6032 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6033}
6034
6035Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(MVT RetVT, Register Op0) {
6036 if (RetVT.SimpleTy != MVT::i16)
6037 return Register();
6038 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6039}
6040
6041Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(MVT RetVT, Register Op0) {
6042 if (RetVT.SimpleTy != MVT::i32)
6043 return Register();
6044 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6045}
6046
6047Register fastEmit_ISD_VECREDUCE_ADD_MVT_v2i64_r(MVT RetVT, Register Op0) {
6048 if (RetVT.SimpleTy != MVT::i64)
6049 return Register();
6050 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6051}
6052
6053Register fastEmit_ISD_VECREDUCE_ADD_r(MVT VT, MVT RetVT, Register Op0) {
6054 switch (VT.SimpleTy) {
6055 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i8_r(RetVT, Op0);
6056 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(RetVT, Op0);
6057 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i16_r(RetVT, Op0);
6058 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(RetVT, Op0);
6059 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(RetVT, Op0);
6060 case MVT::v2i64: return fastEmit_ISD_VECREDUCE_ADD_MVT_v2i64_r(RetVT, Op0);
6061 default: return Register();
6062 }
6063}
6064
6065// FastEmit functions for ISD::VECREDUCE_FADD.
6066
6067Register fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(MVT RetVT, Register Op0) {
6068 if (RetVT.SimpleTy != MVT::f32)
6069 return Register();
6070 return fastEmitInst_r(MachineInstOpcode: AArch64::FADDPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6071}
6072
6073Register fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(MVT RetVT, Register Op0) {
6074 if (RetVT.SimpleTy != MVT::f64)
6075 return Register();
6076 return fastEmitInst_r(MachineInstOpcode: AArch64::FADDPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6077}
6078
6079Register fastEmit_ISD_VECREDUCE_FADD_r(MVT VT, MVT RetVT, Register Op0) {
6080 switch (VT.SimpleTy) {
6081 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(RetVT, Op0);
6082 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(RetVT, Op0);
6083 default: return Register();
6084 }
6085}
6086
6087// FastEmit functions for ISD::VECREDUCE_FMAX.
6088
6089Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f16_r(MVT RetVT, Register Op0) {
6090 if (RetVT.SimpleTy != MVT::f16)
6091 return Register();
6092 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6093 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6094 }
6095 return Register();
6096}
6097
6098Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v8f16_r(MVT RetVT, Register Op0) {
6099 if (RetVT.SimpleTy != MVT::f16)
6100 return Register();
6101 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6102 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6103 }
6104 return Register();
6105}
6106
6107Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f32_r(MVT RetVT, Register Op0) {
6108 if (RetVT.SimpleTy != MVT::f32)
6109 return Register();
6110 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6111}
6112
6113Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f32_r(MVT RetVT, Register Op0) {
6114 if (RetVT.SimpleTy != MVT::f32)
6115 return Register();
6116 if ((Subtarget->isNeonAvailable())) {
6117 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6118 }
6119 return Register();
6120}
6121
6122Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f64_r(MVT RetVT, Register Op0) {
6123 if (RetVT.SimpleTy != MVT::f64)
6124 return Register();
6125 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6126}
6127
6128Register fastEmit_ISD_VECREDUCE_FMAX_r(MVT VT, MVT RetVT, Register Op0) {
6129 switch (VT.SimpleTy) {
6130 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f16_r(RetVT, Op0);
6131 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v8f16_r(RetVT, Op0);
6132 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f32_r(RetVT, Op0);
6133 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f32_r(RetVT, Op0);
6134 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f64_r(RetVT, Op0);
6135 default: return Register();
6136 }
6137}
6138
6139// FastEmit functions for ISD::VECREDUCE_FMAXIMUM.
6140
6141Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f16_r(MVT RetVT, Register Op0) {
6142 if (RetVT.SimpleTy != MVT::f16)
6143 return Register();
6144 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6145 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6146 }
6147 return Register();
6148}
6149
6150Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v8f16_r(MVT RetVT, Register Op0) {
6151 if (RetVT.SimpleTy != MVT::f16)
6152 return Register();
6153 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6154 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6155 }
6156 return Register();
6157}
6158
6159Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f32_r(MVT RetVT, Register Op0) {
6160 if (RetVT.SimpleTy != MVT::f32)
6161 return Register();
6162 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6163}
6164
6165Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f32_r(MVT RetVT, Register Op0) {
6166 if (RetVT.SimpleTy != MVT::f32)
6167 return Register();
6168 if ((Subtarget->isNeonAvailable())) {
6169 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6170 }
6171 return Register();
6172}
6173
6174Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f64_r(MVT RetVT, Register Op0) {
6175 if (RetVT.SimpleTy != MVT::f64)
6176 return Register();
6177 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6178}
6179
6180Register fastEmit_ISD_VECREDUCE_FMAXIMUM_r(MVT VT, MVT RetVT, Register Op0) {
6181 switch (VT.SimpleTy) {
6182 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f16_r(RetVT, Op0);
6183 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v8f16_r(RetVT, Op0);
6184 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f32_r(RetVT, Op0);
6185 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f32_r(RetVT, Op0);
6186 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f64_r(RetVT, Op0);
6187 default: return Register();
6188 }
6189}
6190
6191// FastEmit functions for ISD::VECREDUCE_FMIN.
6192
6193Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f16_r(MVT RetVT, Register Op0) {
6194 if (RetVT.SimpleTy != MVT::f16)
6195 return Register();
6196 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6197 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6198 }
6199 return Register();
6200}
6201
6202Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v8f16_r(MVT RetVT, Register Op0) {
6203 if (RetVT.SimpleTy != MVT::f16)
6204 return Register();
6205 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6206 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6207 }
6208 return Register();
6209}
6210
6211Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f32_r(MVT RetVT, Register Op0) {
6212 if (RetVT.SimpleTy != MVT::f32)
6213 return Register();
6214 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6215}
6216
6217Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f32_r(MVT RetVT, Register Op0) {
6218 if (RetVT.SimpleTy != MVT::f32)
6219 return Register();
6220 if ((Subtarget->isNeonAvailable())) {
6221 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6222 }
6223 return Register();
6224}
6225
6226Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f64_r(MVT RetVT, Register Op0) {
6227 if (RetVT.SimpleTy != MVT::f64)
6228 return Register();
6229 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6230}
6231
6232Register fastEmit_ISD_VECREDUCE_FMIN_r(MVT VT, MVT RetVT, Register Op0) {
6233 switch (VT.SimpleTy) {
6234 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f16_r(RetVT, Op0);
6235 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v8f16_r(RetVT, Op0);
6236 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f32_r(RetVT, Op0);
6237 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f32_r(RetVT, Op0);
6238 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f64_r(RetVT, Op0);
6239 default: return Register();
6240 }
6241}
6242
6243// FastEmit functions for ISD::VECREDUCE_FMINIMUM.
6244
6245Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f16_r(MVT RetVT, Register Op0) {
6246 if (RetVT.SimpleTy != MVT::f16)
6247 return Register();
6248 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6249 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6250 }
6251 return Register();
6252}
6253
6254Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v8f16_r(MVT RetVT, Register Op0) {
6255 if (RetVT.SimpleTy != MVT::f16)
6256 return Register();
6257 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6258 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6259 }
6260 return Register();
6261}
6262
6263Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f32_r(MVT RetVT, Register Op0) {
6264 if (RetVT.SimpleTy != MVT::f32)
6265 return Register();
6266 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6267}
6268
6269Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f32_r(MVT RetVT, Register Op0) {
6270 if (RetVT.SimpleTy != MVT::f32)
6271 return Register();
6272 if ((Subtarget->isNeonAvailable())) {
6273 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6274 }
6275 return Register();
6276}
6277
6278Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f64_r(MVT RetVT, Register Op0) {
6279 if (RetVT.SimpleTy != MVT::f64)
6280 return Register();
6281 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6282}
6283
6284Register fastEmit_ISD_VECREDUCE_FMINIMUM_r(MVT VT, MVT RetVT, Register Op0) {
6285 switch (VT.SimpleTy) {
6286 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f16_r(RetVT, Op0);
6287 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v8f16_r(RetVT, Op0);
6288 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f32_r(RetVT, Op0);
6289 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f32_r(RetVT, Op0);
6290 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f64_r(RetVT, Op0);
6291 default: return Register();
6292 }
6293}
6294
6295// FastEmit functions for ISD::VECREDUCE_SMAX.
6296
6297Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i8_r(MVT RetVT, Register Op0) {
6298 if (RetVT.SimpleTy != MVT::i8)
6299 return Register();
6300 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6301}
6302
6303Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v16i8_r(MVT RetVT, Register Op0) {
6304 if (RetVT.SimpleTy != MVT::i8)
6305 return Register();
6306 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6307}
6308
6309Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i16_r(MVT RetVT, Register Op0) {
6310 if (RetVT.SimpleTy != MVT::i16)
6311 return Register();
6312 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6313}
6314
6315Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i16_r(MVT RetVT, Register Op0) {
6316 if (RetVT.SimpleTy != MVT::i16)
6317 return Register();
6318 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6319}
6320
6321Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i32_r(MVT RetVT, Register Op0) {
6322 if (RetVT.SimpleTy != MVT::i32)
6323 return Register();
6324 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6325}
6326
6327Register fastEmit_ISD_VECREDUCE_SMAX_r(MVT VT, MVT RetVT, Register Op0) {
6328 switch (VT.SimpleTy) {
6329 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i8_r(RetVT, Op0);
6330 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v16i8_r(RetVT, Op0);
6331 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i16_r(RetVT, Op0);
6332 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i16_r(RetVT, Op0);
6333 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i32_r(RetVT, Op0);
6334 default: return Register();
6335 }
6336}
6337
6338// FastEmit functions for ISD::VECREDUCE_SMIN.
6339
6340Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i8_r(MVT RetVT, Register Op0) {
6341 if (RetVT.SimpleTy != MVT::i8)
6342 return Register();
6343 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6344}
6345
6346Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v16i8_r(MVT RetVT, Register Op0) {
6347 if (RetVT.SimpleTy != MVT::i8)
6348 return Register();
6349 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6350}
6351
6352Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i16_r(MVT RetVT, Register Op0) {
6353 if (RetVT.SimpleTy != MVT::i16)
6354 return Register();
6355 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6356}
6357
6358Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i16_r(MVT RetVT, Register Op0) {
6359 if (RetVT.SimpleTy != MVT::i16)
6360 return Register();
6361 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6362}
6363
6364Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i32_r(MVT RetVT, Register Op0) {
6365 if (RetVT.SimpleTy != MVT::i32)
6366 return Register();
6367 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6368}
6369
6370Register fastEmit_ISD_VECREDUCE_SMIN_r(MVT VT, MVT RetVT, Register Op0) {
6371 switch (VT.SimpleTy) {
6372 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i8_r(RetVT, Op0);
6373 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v16i8_r(RetVT, Op0);
6374 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i16_r(RetVT, Op0);
6375 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i16_r(RetVT, Op0);
6376 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i32_r(RetVT, Op0);
6377 default: return Register();
6378 }
6379}
6380
6381// FastEmit functions for ISD::VECREDUCE_UMAX.
6382
6383Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i8_r(MVT RetVT, Register Op0) {
6384 if (RetVT.SimpleTy != MVT::i8)
6385 return Register();
6386 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6387}
6388
6389Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v16i8_r(MVT RetVT, Register Op0) {
6390 if (RetVT.SimpleTy != MVT::i8)
6391 return Register();
6392 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6393}
6394
6395Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i16_r(MVT RetVT, Register Op0) {
6396 if (RetVT.SimpleTy != MVT::i16)
6397 return Register();
6398 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6399}
6400
6401Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i16_r(MVT RetVT, Register Op0) {
6402 if (RetVT.SimpleTy != MVT::i16)
6403 return Register();
6404 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6405}
6406
6407Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i32_r(MVT RetVT, Register Op0) {
6408 if (RetVT.SimpleTy != MVT::i32)
6409 return Register();
6410 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6411}
6412
6413Register fastEmit_ISD_VECREDUCE_UMAX_r(MVT VT, MVT RetVT, Register Op0) {
6414 switch (VT.SimpleTy) {
6415 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i8_r(RetVT, Op0);
6416 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v16i8_r(RetVT, Op0);
6417 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i16_r(RetVT, Op0);
6418 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i16_r(RetVT, Op0);
6419 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i32_r(RetVT, Op0);
6420 default: return Register();
6421 }
6422}
6423
6424// FastEmit functions for ISD::VECREDUCE_UMIN.
6425
6426Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i8_r(MVT RetVT, Register Op0) {
6427 if (RetVT.SimpleTy != MVT::i8)
6428 return Register();
6429 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6430}
6431
6432Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v16i8_r(MVT RetVT, Register Op0) {
6433 if (RetVT.SimpleTy != MVT::i8)
6434 return Register();
6435 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6436}
6437
6438Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i16_r(MVT RetVT, Register Op0) {
6439 if (RetVT.SimpleTy != MVT::i16)
6440 return Register();
6441 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6442}
6443
6444Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i16_r(MVT RetVT, Register Op0) {
6445 if (RetVT.SimpleTy != MVT::i16)
6446 return Register();
6447 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6448}
6449
6450Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i32_r(MVT RetVT, Register Op0) {
6451 if (RetVT.SimpleTy != MVT::i32)
6452 return Register();
6453 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6454}
6455
6456Register fastEmit_ISD_VECREDUCE_UMIN_r(MVT VT, MVT RetVT, Register Op0) {
6457 switch (VT.SimpleTy) {
6458 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i8_r(RetVT, Op0);
6459 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v16i8_r(RetVT, Op0);
6460 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i16_r(RetVT, Op0);
6461 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i16_r(RetVT, Op0);
6462 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i32_r(RetVT, Op0);
6463 default: return Register();
6464 }
6465}
6466
6467// FastEmit functions for ISD::VECTOR_REVERSE.
6468
6469Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(MVT RetVT, Register Op0) {
6470 if (RetVT.SimpleTy != MVT::nxv2i1)
6471 return Register();
6472 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6473 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_D, RC: &AArch64::PPRRegClass, Op0);
6474 }
6475 return Register();
6476}
6477
6478Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(MVT RetVT, Register Op0) {
6479 if (RetVT.SimpleTy != MVT::nxv4i1)
6480 return Register();
6481 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6482 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_S, RC: &AArch64::PPRRegClass, Op0);
6483 }
6484 return Register();
6485}
6486
6487Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(MVT RetVT, Register Op0) {
6488 if (RetVT.SimpleTy != MVT::nxv8i1)
6489 return Register();
6490 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6491 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_H, RC: &AArch64::PPRRegClass, Op0);
6492 }
6493 return Register();
6494}
6495
6496Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(MVT RetVT, Register Op0) {
6497 if (RetVT.SimpleTy != MVT::nxv16i1)
6498 return Register();
6499 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6500 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_B, RC: &AArch64::PPRRegClass, Op0);
6501 }
6502 return Register();
6503}
6504
6505Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
6506 if (RetVT.SimpleTy != MVT::nxv16i8)
6507 return Register();
6508 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6509 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_B, RC: &AArch64::ZPRRegClass, Op0);
6510 }
6511 return Register();
6512}
6513
6514Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
6515 if (RetVT.SimpleTy != MVT::nxv8i16)
6516 return Register();
6517 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6518 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6519 }
6520 return Register();
6521}
6522
6523Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
6524 if (RetVT.SimpleTy != MVT::nxv4i32)
6525 return Register();
6526 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6527 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6528 }
6529 return Register();
6530}
6531
6532Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(MVT RetVT, Register Op0) {
6533 if (RetVT.SimpleTy != MVT::nxv2i64)
6534 return Register();
6535 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6536 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6537 }
6538 return Register();
6539}
6540
6541Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(MVT RetVT, Register Op0) {
6542 if (RetVT.SimpleTy != MVT::nxv2f16)
6543 return Register();
6544 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6545 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6546 }
6547 return Register();
6548}
6549
6550Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(MVT RetVT, Register Op0) {
6551 if (RetVT.SimpleTy != MVT::nxv4f16)
6552 return Register();
6553 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6554 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6555 }
6556 return Register();
6557}
6558
6559Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
6560 if (RetVT.SimpleTy != MVT::nxv8f16)
6561 return Register();
6562 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6563 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6564 }
6565 return Register();
6566}
6567
6568Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(MVT RetVT, Register Op0) {
6569 if (RetVT.SimpleTy != MVT::nxv2bf16)
6570 return Register();
6571 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6572 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6573 }
6574 return Register();
6575}
6576
6577Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(MVT RetVT, Register Op0) {
6578 if (RetVT.SimpleTy != MVT::nxv4bf16)
6579 return Register();
6580 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6581 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6582 }
6583 return Register();
6584}
6585
6586Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(MVT RetVT, Register Op0) {
6587 if (RetVT.SimpleTy != MVT::nxv8bf16)
6588 return Register();
6589 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6590 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6591 }
6592 return Register();
6593}
6594
6595Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(MVT RetVT, Register Op0) {
6596 if (RetVT.SimpleTy != MVT::nxv2f32)
6597 return Register();
6598 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6599 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6600 }
6601 return Register();
6602}
6603
6604Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
6605 if (RetVT.SimpleTy != MVT::nxv4f32)
6606 return Register();
6607 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6608 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6609 }
6610 return Register();
6611}
6612
6613Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
6614 if (RetVT.SimpleTy != MVT::nxv2f64)
6615 return Register();
6616 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6617 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6618 }
6619 return Register();
6620}
6621
6622Register fastEmit_ISD_VECTOR_REVERSE_r(MVT VT, MVT RetVT, Register Op0) {
6623 switch (VT.SimpleTy) {
6624 case MVT::nxv2i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(RetVT, Op0);
6625 case MVT::nxv4i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(RetVT, Op0);
6626 case MVT::nxv8i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(RetVT, Op0);
6627 case MVT::nxv16i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(RetVT, Op0);
6628 case MVT::nxv16i8: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(RetVT, Op0);
6629 case MVT::nxv8i16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(RetVT, Op0);
6630 case MVT::nxv4i32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(RetVT, Op0);
6631 case MVT::nxv2i64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(RetVT, Op0);
6632 case MVT::nxv2f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(RetVT, Op0);
6633 case MVT::nxv4f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(RetVT, Op0);
6634 case MVT::nxv8f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(RetVT, Op0);
6635 case MVT::nxv2bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(RetVT, Op0);
6636 case MVT::nxv4bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(RetVT, Op0);
6637 case MVT::nxv8bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(RetVT, Op0);
6638 case MVT::nxv2f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(RetVT, Op0);
6639 case MVT::nxv4f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(RetVT, Op0);
6640 case MVT::nxv2f64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(RetVT, Op0);
6641 default: return Register();
6642 }
6643}
6644
6645// Top-level FastEmit function.
6646
6647Register fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, Register Op0) override {
6648 switch (Opcode) {
6649 case AArch64ISD::CALL: return fastEmit_AArch64ISD_CALL_r(VT, RetVT, Op0);
6650 case AArch64ISD::COALESCER_BARRIER: return fastEmit_AArch64ISD_COALESCER_BARRIER_r(VT, RetVT, Op0);
6651 case AArch64ISD::DUP: return fastEmit_AArch64ISD_DUP_r(VT, RetVT, Op0);
6652 case AArch64ISD::FCVTXN: return fastEmit_AArch64ISD_FCVTXN_r(VT, RetVT, Op0);
6653 case AArch64ISD::FRECPE: return fastEmit_AArch64ISD_FRECPE_r(VT, RetVT, Op0);
6654 case AArch64ISD::FRSQRTE: return fastEmit_AArch64ISD_FRSQRTE_r(VT, RetVT, Op0);
6655 case AArch64ISD::PROBED_ALLOCA: return fastEmit_AArch64ISD_PROBED_ALLOCA_r(VT, RetVT, Op0);
6656 case AArch64ISD::REV16: return fastEmit_AArch64ISD_REV16_r(VT, RetVT, Op0);
6657 case AArch64ISD::REV32: return fastEmit_AArch64ISD_REV32_r(VT, RetVT, Op0);
6658 case AArch64ISD::REV64: return fastEmit_AArch64ISD_REV64_r(VT, RetVT, Op0);
6659 case AArch64ISD::SADDLP: return fastEmit_AArch64ISD_SADDLP_r(VT, RetVT, Op0);
6660 case AArch64ISD::SITOF: return fastEmit_AArch64ISD_SITOF_r(VT, RetVT, Op0);
6661 case AArch64ISD::SQABS: return fastEmit_AArch64ISD_SQABS_r(VT, RetVT, Op0);
6662 case AArch64ISD::SQNEG: return fastEmit_AArch64ISD_SQNEG_r(VT, RetVT, Op0);
6663 case AArch64ISD::SUNPKHI: return fastEmit_AArch64ISD_SUNPKHI_r(VT, RetVT, Op0);
6664 case AArch64ISD::SUNPKLO: return fastEmit_AArch64ISD_SUNPKLO_r(VT, RetVT, Op0);
6665 case AArch64ISD::UADDLP: return fastEmit_AArch64ISD_UADDLP_r(VT, RetVT, Op0);
6666 case AArch64ISD::UITOF: return fastEmit_AArch64ISD_UITOF_r(VT, RetVT, Op0);
6667 case AArch64ISD::UUNPKHI: return fastEmit_AArch64ISD_UUNPKHI_r(VT, RetVT, Op0);
6668 case AArch64ISD::UUNPKLO: return fastEmit_AArch64ISD_UUNPKLO_r(VT, RetVT, Op0);
6669 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
6670 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
6671 case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0);
6672 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0);
6673 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
6674 case ISD::CTLS: return fastEmit_ISD_CTLS_r(VT, RetVT, Op0);
6675 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
6676 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
6677 case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0);
6678 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
6679 case ISD::FCEIL: return fastEmit_ISD_FCEIL_r(VT, RetVT, Op0);
6680 case ISD::FFLOOR: return fastEmit_ISD_FFLOOR_r(VT, RetVT, Op0);
6681 case ISD::FNEARBYINT: return fastEmit_ISD_FNEARBYINT_r(VT, RetVT, Op0);
6682 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
6683 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
6684 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
6685 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
6686 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
6687 case ISD::FRINT: return fastEmit_ISD_FRINT_r(VT, RetVT, Op0);
6688 case ISD::FROUND: return fastEmit_ISD_FROUND_r(VT, RetVT, Op0);
6689 case ISD::FROUNDEVEN: return fastEmit_ISD_FROUNDEVEN_r(VT, RetVT, Op0);
6690 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
6691 case ISD::FTRUNC: return fastEmit_ISD_FTRUNC_r(VT, RetVT, Op0);
6692 case ISD::LLROUND: return fastEmit_ISD_LLROUND_r(VT, RetVT, Op0);
6693 case ISD::LROUND: return fastEmit_ISD_LROUND_r(VT, RetVT, Op0);
6694 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
6695 case ISD::SPLAT_VECTOR: return fastEmit_ISD_SPLAT_VECTOR_r(VT, RetVT, Op0);
6696 case ISD::STRICT_FCEIL: return fastEmit_ISD_STRICT_FCEIL_r(VT, RetVT, Op0);
6697 case ISD::STRICT_FFLOOR: return fastEmit_ISD_STRICT_FFLOOR_r(VT, RetVT, Op0);
6698 case ISD::STRICT_FNEARBYINT: return fastEmit_ISD_STRICT_FNEARBYINT_r(VT, RetVT, Op0);
6699 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0);
6700 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0);
6701 case ISD::STRICT_FP_TO_SINT: return fastEmit_ISD_STRICT_FP_TO_SINT_r(VT, RetVT, Op0);
6702 case ISD::STRICT_FP_TO_UINT: return fastEmit_ISD_STRICT_FP_TO_UINT_r(VT, RetVT, Op0);
6703 case ISD::STRICT_FRINT: return fastEmit_ISD_STRICT_FRINT_r(VT, RetVT, Op0);
6704 case ISD::STRICT_FROUND: return fastEmit_ISD_STRICT_FROUND_r(VT, RetVT, Op0);
6705 case ISD::STRICT_FROUNDEVEN: return fastEmit_ISD_STRICT_FROUNDEVEN_r(VT, RetVT, Op0);
6706 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0);
6707 case ISD::STRICT_FTRUNC: return fastEmit_ISD_STRICT_FTRUNC_r(VT, RetVT, Op0);
6708 case ISD::STRICT_LLROUND: return fastEmit_ISD_STRICT_LLROUND_r(VT, RetVT, Op0);
6709 case ISD::STRICT_LROUND: return fastEmit_ISD_STRICT_LROUND_r(VT, RetVT, Op0);
6710 case ISD::STRICT_SINT_TO_FP: return fastEmit_ISD_STRICT_SINT_TO_FP_r(VT, RetVT, Op0);
6711 case ISD::STRICT_UINT_TO_FP: return fastEmit_ISD_STRICT_UINT_TO_FP_r(VT, RetVT, Op0);
6712 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
6713 case ISD::TRUNCATE_SSAT_S: return fastEmit_ISD_TRUNCATE_SSAT_S_r(VT, RetVT, Op0);
6714 case ISD::TRUNCATE_SSAT_U: return fastEmit_ISD_TRUNCATE_SSAT_U_r(VT, RetVT, Op0);
6715 case ISD::TRUNCATE_USAT_U: return fastEmit_ISD_TRUNCATE_USAT_U_r(VT, RetVT, Op0);
6716 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
6717 case ISD::VECREDUCE_ADD: return fastEmit_ISD_VECREDUCE_ADD_r(VT, RetVT, Op0);
6718 case ISD::VECREDUCE_FADD: return fastEmit_ISD_VECREDUCE_FADD_r(VT, RetVT, Op0);
6719 case ISD::VECREDUCE_FMAX: return fastEmit_ISD_VECREDUCE_FMAX_r(VT, RetVT, Op0);
6720 case ISD::VECREDUCE_FMAXIMUM: return fastEmit_ISD_VECREDUCE_FMAXIMUM_r(VT, RetVT, Op0);
6721 case ISD::VECREDUCE_FMIN: return fastEmit_ISD_VECREDUCE_FMIN_r(VT, RetVT, Op0);
6722 case ISD::VECREDUCE_FMINIMUM: return fastEmit_ISD_VECREDUCE_FMINIMUM_r(VT, RetVT, Op0);
6723 case ISD::VECREDUCE_SMAX: return fastEmit_ISD_VECREDUCE_SMAX_r(VT, RetVT, Op0);
6724 case ISD::VECREDUCE_SMIN: return fastEmit_ISD_VECREDUCE_SMIN_r(VT, RetVT, Op0);
6725 case ISD::VECREDUCE_UMAX: return fastEmit_ISD_VECREDUCE_UMAX_r(VT, RetVT, Op0);
6726 case ISD::VECREDUCE_UMIN: return fastEmit_ISD_VECREDUCE_UMIN_r(VT, RetVT, Op0);
6727 case ISD::VECTOR_REVERSE: return fastEmit_ISD_VECTOR_REVERSE_r(VT, RetVT, Op0);
6728 default: return Register();
6729 }
6730}
6731
6732// FastEmit functions for AArch64ISD::ADDP.
6733
6734Register fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6735 if (RetVT.SimpleTy != MVT::v8i8)
6736 return Register();
6737 if ((Subtarget->isNeonAvailable())) {
6738 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
6739 }
6740 return Register();
6741}
6742
6743Register fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6744 if (RetVT.SimpleTy != MVT::v16i8)
6745 return Register();
6746 if ((Subtarget->isNeonAvailable())) {
6747 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
6748 }
6749 return Register();
6750}
6751
6752Register fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6753 if (RetVT.SimpleTy != MVT::v4i16)
6754 return Register();
6755 if ((Subtarget->isNeonAvailable())) {
6756 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6757 }
6758 return Register();
6759}
6760
6761Register fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6762 if (RetVT.SimpleTy != MVT::v8i16)
6763 return Register();
6764 if ((Subtarget->isNeonAvailable())) {
6765 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
6766 }
6767 return Register();
6768}
6769
6770Register fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6771 if (RetVT.SimpleTy != MVT::v2i32)
6772 return Register();
6773 if ((Subtarget->isNeonAvailable())) {
6774 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6775 }
6776 return Register();
6777}
6778
6779Register fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6780 if (RetVT.SimpleTy != MVT::v4i32)
6781 return Register();
6782 if ((Subtarget->isNeonAvailable())) {
6783 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
6784 }
6785 return Register();
6786}
6787
6788Register fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6789 if (RetVT.SimpleTy != MVT::v2i64)
6790 return Register();
6791 if ((Subtarget->isNeonAvailable())) {
6792 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
6793 }
6794 return Register();
6795}
6796
6797Register fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
6798 if (RetVT.SimpleTy != MVT::v4f16)
6799 return Register();
6800 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6801 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6802 }
6803 return Register();
6804}
6805
6806Register fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
6807 if (RetVT.SimpleTy != MVT::v8f16)
6808 return Register();
6809 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6810 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
6811 }
6812 return Register();
6813}
6814
6815Register fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
6816 if (RetVT.SimpleTy != MVT::v2f32)
6817 return Register();
6818 if ((Subtarget->isNeonAvailable())) {
6819 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6820 }
6821 return Register();
6822}
6823
6824Register fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
6825 if (RetVT.SimpleTy != MVT::v4f32)
6826 return Register();
6827 if ((Subtarget->isNeonAvailable())) {
6828 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
6829 }
6830 return Register();
6831}
6832
6833Register fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
6834 if (RetVT.SimpleTy != MVT::v2f64)
6835 return Register();
6836 if ((Subtarget->isNeonAvailable())) {
6837 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
6838 }
6839 return Register();
6840}
6841
6842Register fastEmit_AArch64ISD_ADDP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6843 switch (VT.SimpleTy) {
6844 case MVT::v8i8: return fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(RetVT, Op0, Op1);
6845 case MVT::v16i8: return fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(RetVT, Op0, Op1);
6846 case MVT::v4i16: return fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(RetVT, Op0, Op1);
6847 case MVT::v8i16: return fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(RetVT, Op0, Op1);
6848 case MVT::v2i32: return fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(RetVT, Op0, Op1);
6849 case MVT::v4i32: return fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(RetVT, Op0, Op1);
6850 case MVT::v2i64: return fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(RetVT, Op0, Op1);
6851 case MVT::v4f16: return fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(RetVT, Op0, Op1);
6852 case MVT::v8f16: return fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(RetVT, Op0, Op1);
6853 case MVT::v2f32: return fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(RetVT, Op0, Op1);
6854 case MVT::v4f32: return fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(RetVT, Op0, Op1);
6855 case MVT::v2f64: return fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(RetVT, Op0, Op1);
6856 default: return Register();
6857 }
6858}
6859
6860// FastEmit functions for AArch64ISD::BIC.
6861
6862Register fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6863 if (RetVT.SimpleTy != MVT::nxv16i8)
6864 return Register();
6865 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6866 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6867 }
6868 return Register();
6869}
6870
6871Register fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6872 if (RetVT.SimpleTy != MVT::nxv8i16)
6873 return Register();
6874 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6875 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6876 }
6877 return Register();
6878}
6879
6880Register fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6881 if (RetVT.SimpleTy != MVT::nxv4i32)
6882 return Register();
6883 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6884 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6885 }
6886 return Register();
6887}
6888
6889Register fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6890 if (RetVT.SimpleTy != MVT::nxv2i64)
6891 return Register();
6892 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6893 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6894 }
6895 return Register();
6896}
6897
6898Register fastEmit_AArch64ISD_BIC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6899 switch (VT.SimpleTy) {
6900 case MVT::nxv16i8: return fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(RetVT, Op0, Op1);
6901 case MVT::nxv8i16: return fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(RetVT, Op0, Op1);
6902 case MVT::nxv4i32: return fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(RetVT, Op0, Op1);
6903 case MVT::nxv2i64: return fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(RetVT, Op0, Op1);
6904 default: return Register();
6905 }
6906}
6907
6908// FastEmit functions for AArch64ISD::FCMEQ.
6909
6910Register fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6911 if (RetVT.SimpleTy != MVT::i32)
6912 return Register();
6913 if ((Subtarget->isNeonAvailable())) {
6914 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ32, RC: &AArch64::FPR32RegClass, Op0, Op1);
6915 }
6916 return Register();
6917}
6918
6919Register fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6920 if (RetVT.SimpleTy != MVT::i64)
6921 return Register();
6922 if ((Subtarget->isNeonAvailable())) {
6923 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ64, RC: &AArch64::FPR64RegClass, Op0, Op1);
6924 }
6925 return Register();
6926}
6927
6928Register fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
6929 if (RetVT.SimpleTy != MVT::v4i16)
6930 return Register();
6931 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6932 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6933 }
6934 return Register();
6935}
6936
6937Register fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
6938 if (RetVT.SimpleTy != MVT::v8i16)
6939 return Register();
6940 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6941 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
6942 }
6943 return Register();
6944}
6945
6946Register fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
6947 if (RetVT.SimpleTy != MVT::v2i32)
6948 return Register();
6949 if ((Subtarget->isNeonAvailable())) {
6950 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6951 }
6952 return Register();
6953}
6954
6955Register fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
6956 if (RetVT.SimpleTy != MVT::v4i32)
6957 return Register();
6958 if ((Subtarget->isNeonAvailable())) {
6959 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
6960 }
6961 return Register();
6962}
6963
6964Register fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
6965 if (RetVT.SimpleTy != MVT::v1i64)
6966 return Register();
6967 if ((Subtarget->isNeonAvailable())) {
6968 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ64, RC: &AArch64::FPR64RegClass, Op0, Op1);
6969 }
6970 return Register();
6971}
6972
6973Register fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
6974 if (RetVT.SimpleTy != MVT::v2i64)
6975 return Register();
6976 if ((Subtarget->isNeonAvailable())) {
6977 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
6978 }
6979 return Register();
6980}
6981
6982Register fastEmit_AArch64ISD_FCMEQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6983 switch (VT.SimpleTy) {
6984 case MVT::f32: return fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(RetVT, Op0, Op1);
6985 case MVT::f64: return fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(RetVT, Op0, Op1);
6986 case MVT::v4f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(RetVT, Op0, Op1);
6987 case MVT::v8f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(RetVT, Op0, Op1);
6988 case MVT::v2f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(RetVT, Op0, Op1);
6989 case MVT::v4f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(RetVT, Op0, Op1);
6990 case MVT::v1f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(RetVT, Op0, Op1);
6991 case MVT::v2f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(RetVT, Op0, Op1);
6992 default: return Register();
6993 }
6994}
6995
6996// FastEmit functions for AArch64ISD::FCMGE.
6997
6998Register fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6999 if (RetVT.SimpleTy != MVT::i32)
7000 return Register();
7001 if ((Subtarget->isNeonAvailable())) {
7002 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7003 }
7004 return Register();
7005}
7006
7007Register fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7008 if (RetVT.SimpleTy != MVT::i64)
7009 return Register();
7010 if ((Subtarget->isNeonAvailable())) {
7011 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7012 }
7013 return Register();
7014}
7015
7016Register fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7017 if (RetVT.SimpleTy != MVT::v4i16)
7018 return Register();
7019 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7020 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7021 }
7022 return Register();
7023}
7024
7025Register fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7026 if (RetVT.SimpleTy != MVT::v8i16)
7027 return Register();
7028 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7029 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7030 }
7031 return Register();
7032}
7033
7034Register fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7035 if (RetVT.SimpleTy != MVT::v2i32)
7036 return Register();
7037 if ((Subtarget->isNeonAvailable())) {
7038 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7039 }
7040 return Register();
7041}
7042
7043Register fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7044 if (RetVT.SimpleTy != MVT::v4i32)
7045 return Register();
7046 if ((Subtarget->isNeonAvailable())) {
7047 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7048 }
7049 return Register();
7050}
7051
7052Register fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7053 if (RetVT.SimpleTy != MVT::v1i64)
7054 return Register();
7055 if ((Subtarget->isNeonAvailable())) {
7056 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7057 }
7058 return Register();
7059}
7060
7061Register fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7062 if (RetVT.SimpleTy != MVT::v2i64)
7063 return Register();
7064 if ((Subtarget->isNeonAvailable())) {
7065 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7066 }
7067 return Register();
7068}
7069
7070Register fastEmit_AArch64ISD_FCMGE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7071 switch (VT.SimpleTy) {
7072 case MVT::f32: return fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(RetVT, Op0, Op1);
7073 case MVT::f64: return fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(RetVT, Op0, Op1);
7074 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(RetVT, Op0, Op1);
7075 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(RetVT, Op0, Op1);
7076 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(RetVT, Op0, Op1);
7077 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(RetVT, Op0, Op1);
7078 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(RetVT, Op0, Op1);
7079 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(RetVT, Op0, Op1);
7080 default: return Register();
7081 }
7082}
7083
7084// FastEmit functions for AArch64ISD::FCMGT.
7085
7086Register fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7087 if (RetVT.SimpleTy != MVT::i32)
7088 return Register();
7089 if ((Subtarget->isNeonAvailable())) {
7090 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7091 }
7092 return Register();
7093}
7094
7095Register fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7096 if (RetVT.SimpleTy != MVT::i64)
7097 return Register();
7098 if ((Subtarget->isNeonAvailable())) {
7099 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7100 }
7101 return Register();
7102}
7103
7104Register fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7105 if (RetVT.SimpleTy != MVT::v4i16)
7106 return Register();
7107 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7108 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7109 }
7110 return Register();
7111}
7112
7113Register fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7114 if (RetVT.SimpleTy != MVT::v8i16)
7115 return Register();
7116 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7117 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7118 }
7119 return Register();
7120}
7121
7122Register fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7123 if (RetVT.SimpleTy != MVT::v2i32)
7124 return Register();
7125 if ((Subtarget->isNeonAvailable())) {
7126 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7127 }
7128 return Register();
7129}
7130
7131Register fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7132 if (RetVT.SimpleTy != MVT::v4i32)
7133 return Register();
7134 if ((Subtarget->isNeonAvailable())) {
7135 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7136 }
7137 return Register();
7138}
7139
7140Register fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7141 if (RetVT.SimpleTy != MVT::v1i64)
7142 return Register();
7143 if ((Subtarget->isNeonAvailable())) {
7144 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7145 }
7146 return Register();
7147}
7148
7149Register fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7150 if (RetVT.SimpleTy != MVT::v2i64)
7151 return Register();
7152 if ((Subtarget->isNeonAvailable())) {
7153 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7154 }
7155 return Register();
7156}
7157
7158Register fastEmit_AArch64ISD_FCMGT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7159 switch (VT.SimpleTy) {
7160 case MVT::f32: return fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(RetVT, Op0, Op1);
7161 case MVT::f64: return fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(RetVT, Op0, Op1);
7162 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(RetVT, Op0, Op1);
7163 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(RetVT, Op0, Op1);
7164 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(RetVT, Op0, Op1);
7165 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(RetVT, Op0, Op1);
7166 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(RetVT, Op0, Op1);
7167 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(RetVT, Op0, Op1);
7168 default: return Register();
7169 }
7170}
7171
7172// FastEmit functions for AArch64ISD::FCMP.
7173
7174Register fastEmit_AArch64ISD_FCMP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
7175 if (RetVT.SimpleTy != MVT::i32)
7176 return Register();
7177 if ((Subtarget->hasFullFP16())) {
7178 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
7179 }
7180 return Register();
7181}
7182
7183Register fastEmit_AArch64ISD_FCMP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7184 if (RetVT.SimpleTy != MVT::i32)
7185 return Register();
7186 if ((Subtarget->hasFPARMv8())) {
7187 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
7188 }
7189 return Register();
7190}
7191
7192Register fastEmit_AArch64ISD_FCMP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7193 if (RetVT.SimpleTy != MVT::i32)
7194 return Register();
7195 if ((Subtarget->hasFPARMv8())) {
7196 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
7197 }
7198 return Register();
7199}
7200
7201Register fastEmit_AArch64ISD_FCMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7202 switch (VT.SimpleTy) {
7203 case MVT::f16: return fastEmit_AArch64ISD_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
7204 case MVT::f32: return fastEmit_AArch64ISD_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
7205 case MVT::f64: return fastEmit_AArch64ISD_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
7206 default: return Register();
7207 }
7208}
7209
7210// FastEmit functions for AArch64ISD::FRECPS.
7211
7212Register fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7213 if (RetVT.SimpleTy != MVT::f32)
7214 return Register();
7215 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7216}
7217
7218Register fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7219 if (RetVT.SimpleTy != MVT::f64)
7220 return Register();
7221 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7222}
7223
7224Register fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7225 if (RetVT.SimpleTy != MVT::v2f32)
7226 return Register();
7227 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7228}
7229
7230Register fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7231 if (RetVT.SimpleTy != MVT::v4f32)
7232 return Register();
7233 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7234}
7235
7236Register fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7237 if (RetVT.SimpleTy != MVT::v2f64)
7238 return Register();
7239 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7240}
7241
7242Register fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7243 if (RetVT.SimpleTy != MVT::nxv8f16)
7244 return Register();
7245 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7246 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7247 }
7248 return Register();
7249}
7250
7251Register fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7252 if (RetVT.SimpleTy != MVT::nxv4f32)
7253 return Register();
7254 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7255 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7256 }
7257 return Register();
7258}
7259
7260Register fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7261 if (RetVT.SimpleTy != MVT::nxv2f64)
7262 return Register();
7263 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7264 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7265 }
7266 return Register();
7267}
7268
7269Register fastEmit_AArch64ISD_FRECPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7270 switch (VT.SimpleTy) {
7271 case MVT::f32: return fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(RetVT, Op0, Op1);
7272 case MVT::f64: return fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(RetVT, Op0, Op1);
7273 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(RetVT, Op0, Op1);
7274 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(RetVT, Op0, Op1);
7275 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(RetVT, Op0, Op1);
7276 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(RetVT, Op0, Op1);
7277 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(RetVT, Op0, Op1);
7278 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(RetVT, Op0, Op1);
7279 default: return Register();
7280 }
7281}
7282
7283// FastEmit functions for AArch64ISD::FRSQRTS.
7284
7285Register fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7286 if (RetVT.SimpleTy != MVT::f32)
7287 return Register();
7288 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7289}
7290
7291Register fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7292 if (RetVT.SimpleTy != MVT::f64)
7293 return Register();
7294 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7295}
7296
7297Register fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7298 if (RetVT.SimpleTy != MVT::v2f32)
7299 return Register();
7300 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7301}
7302
7303Register fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7304 if (RetVT.SimpleTy != MVT::v4f32)
7305 return Register();
7306 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7307}
7308
7309Register fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7310 if (RetVT.SimpleTy != MVT::v2f64)
7311 return Register();
7312 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7313}
7314
7315Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7316 if (RetVT.SimpleTy != MVT::nxv8f16)
7317 return Register();
7318 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7319 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7320 }
7321 return Register();
7322}
7323
7324Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7325 if (RetVT.SimpleTy != MVT::nxv4f32)
7326 return Register();
7327 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7328 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7329 }
7330 return Register();
7331}
7332
7333Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7334 if (RetVT.SimpleTy != MVT::nxv2f64)
7335 return Register();
7336 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7337 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7338 }
7339 return Register();
7340}
7341
7342Register fastEmit_AArch64ISD_FRSQRTS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7343 switch (VT.SimpleTy) {
7344 case MVT::f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(RetVT, Op0, Op1);
7345 case MVT::f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(RetVT, Op0, Op1);
7346 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(RetVT, Op0, Op1);
7347 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(RetVT, Op0, Op1);
7348 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(RetVT, Op0, Op1);
7349 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(RetVT, Op0, Op1);
7350 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(RetVT, Op0, Op1);
7351 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(RetVT, Op0, Op1);
7352 default: return Register();
7353 }
7354}
7355
7356// FastEmit functions for AArch64ISD::PMULL.
7357
7358Register fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7359 if (RetVT.SimpleTy != MVT::v8i16)
7360 return Register();
7361 if ((Subtarget->isNeonAvailable())) {
7362 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULLv8i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
7363 }
7364 return Register();
7365}
7366
7367Register fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7368 if (RetVT.SimpleTy != MVT::v16i8)
7369 return Register();
7370 if ((Subtarget->hasAES())) {
7371 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULLv1i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7372 }
7373 return Register();
7374}
7375
7376Register fastEmit_AArch64ISD_PMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7377 switch (VT.SimpleTy) {
7378 case MVT::v8i8: return fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
7379 case MVT::v1i64: return fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(RetVT, Op0, Op1);
7380 default: return Register();
7381 }
7382}
7383
7384// FastEmit functions for AArch64ISD::PTEST.
7385
7386Register fastEmit_AArch64ISD_PTEST_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7387 if (RetVT.SimpleTy != MVT::i32)
7388 return Register();
7389 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7390 return fastEmitInst_rr(MachineInstOpcode: AArch64::PTEST_PP, RC: &AArch64::PPRRegClass, Op0, Op1);
7391 }
7392 return Register();
7393}
7394
7395Register fastEmit_AArch64ISD_PTEST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7396 switch (VT.SimpleTy) {
7397 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7398 default: return Register();
7399 }
7400}
7401
7402// FastEmit functions for AArch64ISD::PTEST_ANY.
7403
7404Register fastEmit_AArch64ISD_PTEST_ANY_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7405 if (RetVT.SimpleTy != MVT::i32)
7406 return Register();
7407 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7408 return fastEmitInst_rr(MachineInstOpcode: AArch64::PTEST_PP_ANY, RC: &AArch64::PPRRegClass, Op0, Op1);
7409 }
7410 return Register();
7411}
7412
7413Register fastEmit_AArch64ISD_PTEST_ANY_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7414 switch (VT.SimpleTy) {
7415 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_ANY_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7416 default: return Register();
7417 }
7418}
7419
7420// FastEmit functions for AArch64ISD::PTEST_FIRST.
7421
7422Register fastEmit_AArch64ISD_PTEST_FIRST_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7423 if (RetVT.SimpleTy != MVT::i32)
7424 return Register();
7425 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7426 return fastEmitInst_rr(MachineInstOpcode: AArch64::PTEST_PP_FIRST, RC: &AArch64::PPRRegClass, Op0, Op1);
7427 }
7428 return Register();
7429}
7430
7431Register fastEmit_AArch64ISD_PTEST_FIRST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7432 switch (VT.SimpleTy) {
7433 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_FIRST_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7434 default: return Register();
7435 }
7436}
7437
7438// FastEmit functions for AArch64ISD::SMULL.
7439
7440Register fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7441 if (RetVT.SimpleTy != MVT::v8i16)
7442 return Register();
7443 if ((Subtarget->isNeonAvailable())) {
7444 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv8i8_v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7445 }
7446 return Register();
7447}
7448
7449Register fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7450 if (RetVT.SimpleTy != MVT::v4i32)
7451 return Register();
7452 if ((Subtarget->isNeonAvailable())) {
7453 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv4i16_v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7454 }
7455 return Register();
7456}
7457
7458Register fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7459 if (RetVT.SimpleTy != MVT::v2i64)
7460 return Register();
7461 if ((Subtarget->isNeonAvailable())) {
7462 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv2i32_v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7463 }
7464 return Register();
7465}
7466
7467Register fastEmit_AArch64ISD_SMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7468 switch (VT.SimpleTy) {
7469 case MVT::v8i8: return fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
7470 case MVT::v4i16: return fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(RetVT, Op0, Op1);
7471 case MVT::v2i32: return fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(RetVT, Op0, Op1);
7472 default: return Register();
7473 }
7474}
7475
7476// FastEmit functions for AArch64ISD::SQADD.
7477
7478Register fastEmit_AArch64ISD_SQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7479 if (RetVT.SimpleTy != MVT::f32)
7480 return Register();
7481 if ((Subtarget->isNeonAvailable())) {
7482 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7483 }
7484 return Register();
7485}
7486
7487Register fastEmit_AArch64ISD_SQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7488 if (RetVT.SimpleTy != MVT::f64)
7489 return Register();
7490 if ((Subtarget->isNeonAvailable())) {
7491 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7492 }
7493 return Register();
7494}
7495
7496Register fastEmit_AArch64ISD_SQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7497 switch (VT.SimpleTy) {
7498 case MVT::f32: return fastEmit_AArch64ISD_SQADD_MVT_f32_rr(RetVT, Op0, Op1);
7499 case MVT::f64: return fastEmit_AArch64ISD_SQADD_MVT_f64_rr(RetVT, Op0, Op1);
7500 default: return Register();
7501 }
7502}
7503
7504// FastEmit functions for AArch64ISD::SQDMULH.
7505
7506Register fastEmit_AArch64ISD_SQDMULH_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7507 if (RetVT.SimpleTy != MVT::f32)
7508 return Register();
7509 if ((Subtarget->isNeonAvailable())) {
7510 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7511 }
7512 return Register();
7513}
7514
7515Register fastEmit_AArch64ISD_SQDMULH_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7516 if (RetVT.SimpleTy != MVT::v4i16)
7517 return Register();
7518 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7519}
7520
7521Register fastEmit_AArch64ISD_SQDMULH_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7522 if (RetVT.SimpleTy != MVT::v8i16)
7523 return Register();
7524 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7525}
7526
7527Register fastEmit_AArch64ISD_SQDMULH_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7528 if (RetVT.SimpleTy != MVT::v2i32)
7529 return Register();
7530 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7531}
7532
7533Register fastEmit_AArch64ISD_SQDMULH_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7534 if (RetVT.SimpleTy != MVT::v4i32)
7535 return Register();
7536 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7537}
7538
7539Register fastEmit_AArch64ISD_SQDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7540 switch (VT.SimpleTy) {
7541 case MVT::f32: return fastEmit_AArch64ISD_SQDMULH_MVT_f32_rr(RetVT, Op0, Op1);
7542 case MVT::v4i16: return fastEmit_AArch64ISD_SQDMULH_MVT_v4i16_rr(RetVT, Op0, Op1);
7543 case MVT::v8i16: return fastEmit_AArch64ISD_SQDMULH_MVT_v8i16_rr(RetVT, Op0, Op1);
7544 case MVT::v2i32: return fastEmit_AArch64ISD_SQDMULH_MVT_v2i32_rr(RetVT, Op0, Op1);
7545 case MVT::v4i32: return fastEmit_AArch64ISD_SQDMULH_MVT_v4i32_rr(RetVT, Op0, Op1);
7546 default: return Register();
7547 }
7548}
7549
7550// FastEmit functions for AArch64ISD::SQDMULL.
7551
7552Register fastEmit_AArch64ISD_SQDMULL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7553 if (RetVT.SimpleTy != MVT::f64)
7554 return Register();
7555 if ((Subtarget->isNeonAvailable())) {
7556 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULLi32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7557 }
7558 return Register();
7559}
7560
7561Register fastEmit_AArch64ISD_SQDMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7562 switch (VT.SimpleTy) {
7563 case MVT::f32: return fastEmit_AArch64ISD_SQDMULL_MVT_f32_rr(RetVT, Op0, Op1);
7564 default: return Register();
7565 }
7566}
7567
7568// FastEmit functions for AArch64ISD::SQRDMULH.
7569
7570Register fastEmit_AArch64ISD_SQRDMULH_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7571 if (RetVT.SimpleTy != MVT::f32)
7572 return Register();
7573 if ((Subtarget->isNeonAvailable())) {
7574 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRDMULHv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7575 }
7576 return Register();
7577}
7578
7579Register fastEmit_AArch64ISD_SQRDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7580 switch (VT.SimpleTy) {
7581 case MVT::f32: return fastEmit_AArch64ISD_SQRDMULH_MVT_f32_rr(RetVT, Op0, Op1);
7582 default: return Register();
7583 }
7584}
7585
7586// FastEmit functions for AArch64ISD::SQRSHL.
7587
7588Register fastEmit_AArch64ISD_SQRSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7589 if (RetVT.SimpleTy != MVT::f32)
7590 return Register();
7591 if ((Subtarget->isNeonAvailable())) {
7592 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7593 }
7594 return Register();
7595}
7596
7597Register fastEmit_AArch64ISD_SQRSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7598 if (RetVT.SimpleTy != MVT::f64)
7599 return Register();
7600 if ((Subtarget->isNeonAvailable())) {
7601 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7602 }
7603 return Register();
7604}
7605
7606Register fastEmit_AArch64ISD_SQRSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7607 switch (VT.SimpleTy) {
7608 case MVT::f32: return fastEmit_AArch64ISD_SQRSHL_MVT_f32_rr(RetVT, Op0, Op1);
7609 case MVT::f64: return fastEmit_AArch64ISD_SQRSHL_MVT_f64_rr(RetVT, Op0, Op1);
7610 default: return Register();
7611 }
7612}
7613
7614// FastEmit functions for AArch64ISD::SQSHL.
7615
7616Register fastEmit_AArch64ISD_SQSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7617 if (RetVT.SimpleTy != MVT::f32)
7618 return Register();
7619 if ((Subtarget->isNeonAvailable())) {
7620 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7621 }
7622 return Register();
7623}
7624
7625Register fastEmit_AArch64ISD_SQSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7626 if (RetVT.SimpleTy != MVT::f64)
7627 return Register();
7628 if ((Subtarget->isNeonAvailable())) {
7629 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7630 }
7631 return Register();
7632}
7633
7634Register fastEmit_AArch64ISD_SQSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7635 switch (VT.SimpleTy) {
7636 case MVT::f32: return fastEmit_AArch64ISD_SQSHL_MVT_f32_rr(RetVT, Op0, Op1);
7637 case MVT::f64: return fastEmit_AArch64ISD_SQSHL_MVT_f64_rr(RetVT, Op0, Op1);
7638 default: return Register();
7639 }
7640}
7641
7642// FastEmit functions for AArch64ISD::SQSUB.
7643
7644Register fastEmit_AArch64ISD_SQSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7645 if (RetVT.SimpleTy != MVT::f32)
7646 return Register();
7647 if ((Subtarget->isNeonAvailable())) {
7648 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7649 }
7650 return Register();
7651}
7652
7653Register fastEmit_AArch64ISD_SQSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7654 if (RetVT.SimpleTy != MVT::f64)
7655 return Register();
7656 if ((Subtarget->isNeonAvailable())) {
7657 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7658 }
7659 return Register();
7660}
7661
7662Register fastEmit_AArch64ISD_SQSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7663 switch (VT.SimpleTy) {
7664 case MVT::f32: return fastEmit_AArch64ISD_SQSUB_MVT_f32_rr(RetVT, Op0, Op1);
7665 case MVT::f64: return fastEmit_AArch64ISD_SQSUB_MVT_f64_rr(RetVT, Op0, Op1);
7666 default: return Register();
7667 }
7668}
7669
7670// FastEmit functions for AArch64ISD::STRICT_FCMP.
7671
7672Register fastEmit_AArch64ISD_STRICT_FCMP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
7673 if (RetVT.SimpleTy != MVT::i32)
7674 return Register();
7675 if ((Subtarget->hasFullFP16())) {
7676 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
7677 }
7678 return Register();
7679}
7680
7681Register fastEmit_AArch64ISD_STRICT_FCMP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7682 if (RetVT.SimpleTy != MVT::i32)
7683 return Register();
7684 if ((Subtarget->hasFPARMv8())) {
7685 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
7686 }
7687 return Register();
7688}
7689
7690Register fastEmit_AArch64ISD_STRICT_FCMP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7691 if (RetVT.SimpleTy != MVT::i32)
7692 return Register();
7693 if ((Subtarget->hasFPARMv8())) {
7694 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
7695 }
7696 return Register();
7697}
7698
7699Register fastEmit_AArch64ISD_STRICT_FCMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7700 switch (VT.SimpleTy) {
7701 case MVT::f16: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
7702 case MVT::f32: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
7703 case MVT::f64: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
7704 default: return Register();
7705 }
7706}
7707
7708// FastEmit functions for AArch64ISD::STRICT_FCMPE.
7709
7710Register fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
7711 if (RetVT.SimpleTy != MVT::i32)
7712 return Register();
7713 if ((Subtarget->hasFullFP16())) {
7714 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPEHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
7715 }
7716 return Register();
7717}
7718
7719Register fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7720 if (RetVT.SimpleTy != MVT::i32)
7721 return Register();
7722 if ((Subtarget->hasFPARMv8())) {
7723 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPESrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
7724 }
7725 return Register();
7726}
7727
7728Register fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7729 if (RetVT.SimpleTy != MVT::i32)
7730 return Register();
7731 if ((Subtarget->hasFPARMv8())) {
7732 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPEDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
7733 }
7734 return Register();
7735}
7736
7737Register fastEmit_AArch64ISD_STRICT_FCMPE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7738 switch (VT.SimpleTy) {
7739 case MVT::f16: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f16_rr(RetVT, Op0, Op1);
7740 case MVT::f32: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f32_rr(RetVT, Op0, Op1);
7741 case MVT::f64: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f64_rr(RetVT, Op0, Op1);
7742 default: return Register();
7743 }
7744}
7745
7746// FastEmit functions for AArch64ISD::SUQADD.
7747
7748Register fastEmit_AArch64ISD_SUQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7749 if (RetVT.SimpleTy != MVT::f32)
7750 return Register();
7751 if ((Subtarget->isNeonAvailable())) {
7752 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7753 }
7754 return Register();
7755}
7756
7757Register fastEmit_AArch64ISD_SUQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7758 if (RetVT.SimpleTy != MVT::f64)
7759 return Register();
7760 if ((Subtarget->isNeonAvailable())) {
7761 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7762 }
7763 return Register();
7764}
7765
7766Register fastEmit_AArch64ISD_SUQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7767 switch (VT.SimpleTy) {
7768 case MVT::f32: return fastEmit_AArch64ISD_SUQADD_MVT_f32_rr(RetVT, Op0, Op1);
7769 case MVT::f64: return fastEmit_AArch64ISD_SUQADD_MVT_f64_rr(RetVT, Op0, Op1);
7770 default: return Register();
7771 }
7772}
7773
7774// FastEmit functions for AArch64ISD::TBL.
7775
7776Register fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7777 if (RetVT.SimpleTy != MVT::nxv16i8)
7778 return Register();
7779 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7780 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
7781 }
7782 return Register();
7783}
7784
7785Register fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7786 if (RetVT.SimpleTy != MVT::nxv8i16)
7787 return Register();
7788 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7789 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7790 }
7791 return Register();
7792}
7793
7794Register fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7795 if (RetVT.SimpleTy != MVT::nxv4i32)
7796 return Register();
7797 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7798 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7799 }
7800 return Register();
7801}
7802
7803Register fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7804 if (RetVT.SimpleTy != MVT::nxv2i64)
7805 return Register();
7806 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7807 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7808 }
7809 return Register();
7810}
7811
7812Register fastEmit_AArch64ISD_TBL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7813 switch (VT.SimpleTy) {
7814 case MVT::nxv16i8: return fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(RetVT, Op0, Op1);
7815 case MVT::nxv8i16: return fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(RetVT, Op0, Op1);
7816 case MVT::nxv4i32: return fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(RetVT, Op0, Op1);
7817 case MVT::nxv2i64: return fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(RetVT, Op0, Op1);
7818 default: return Register();
7819 }
7820}
7821
7822// FastEmit functions for AArch64ISD::TRN1.
7823
7824Register fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7825 if (RetVT.SimpleTy != MVT::v8i8)
7826 return Register();
7827 if ((Subtarget->isNeonAvailable())) {
7828 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
7829 }
7830 return Register();
7831}
7832
7833Register fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7834 if (RetVT.SimpleTy != MVT::v16i8)
7835 return Register();
7836 if ((Subtarget->isNeonAvailable())) {
7837 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
7838 }
7839 return Register();
7840}
7841
7842Register fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7843 if (RetVT.SimpleTy != MVT::v4i16)
7844 return Register();
7845 if ((Subtarget->isNeonAvailable())) {
7846 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7847 }
7848 return Register();
7849}
7850
7851Register fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7852 if (RetVT.SimpleTy != MVT::v8i16)
7853 return Register();
7854 if ((Subtarget->isNeonAvailable())) {
7855 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7856 }
7857 return Register();
7858}
7859
7860Register fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7861 if (RetVT.SimpleTy != MVT::v2i32)
7862 return Register();
7863 if ((Subtarget->isNeonAvailable())) {
7864 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7865 }
7866 return Register();
7867}
7868
7869Register fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7870 if (RetVT.SimpleTy != MVT::v4i32)
7871 return Register();
7872 if ((Subtarget->isNeonAvailable())) {
7873 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7874 }
7875 return Register();
7876}
7877
7878Register fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7879 if (RetVT.SimpleTy != MVT::v2i64)
7880 return Register();
7881 if ((Subtarget->isNeonAvailable())) {
7882 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7883 }
7884 return Register();
7885}
7886
7887Register fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7888 if (RetVT.SimpleTy != MVT::v4f16)
7889 return Register();
7890 if ((Subtarget->isNeonAvailable())) {
7891 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7892 }
7893 return Register();
7894}
7895
7896Register fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7897 if (RetVT.SimpleTy != MVT::v8f16)
7898 return Register();
7899 if ((Subtarget->isNeonAvailable())) {
7900 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7901 }
7902 return Register();
7903}
7904
7905Register fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
7906 if (RetVT.SimpleTy != MVT::v4bf16)
7907 return Register();
7908 if ((Subtarget->isNeonAvailable())) {
7909 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7910 }
7911 return Register();
7912}
7913
7914Register fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
7915 if (RetVT.SimpleTy != MVT::v8bf16)
7916 return Register();
7917 if ((Subtarget->isNeonAvailable())) {
7918 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7919 }
7920 return Register();
7921}
7922
7923Register fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7924 if (RetVT.SimpleTy != MVT::v2f32)
7925 return Register();
7926 if ((Subtarget->isNeonAvailable())) {
7927 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7928 }
7929 return Register();
7930}
7931
7932Register fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7933 if (RetVT.SimpleTy != MVT::v4f32)
7934 return Register();
7935 if ((Subtarget->isNeonAvailable())) {
7936 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7937 }
7938 return Register();
7939}
7940
7941Register fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7942 if (RetVT.SimpleTy != MVT::v2f64)
7943 return Register();
7944 if ((Subtarget->isNeonAvailable())) {
7945 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7946 }
7947 return Register();
7948}
7949
7950Register fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
7951 if (RetVT.SimpleTy != MVT::nxv2i1)
7952 return Register();
7953 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7954 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
7955 }
7956 return Register();
7957}
7958
7959Register fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
7960 if (RetVT.SimpleTy != MVT::nxv4i1)
7961 return Register();
7962 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7963 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
7964 }
7965 return Register();
7966}
7967
7968Register fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
7969 if (RetVT.SimpleTy != MVT::nxv8i1)
7970 return Register();
7971 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7972 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
7973 }
7974 return Register();
7975}
7976
7977Register fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7978 if (RetVT.SimpleTy != MVT::nxv16i1)
7979 return Register();
7980 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7981 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
7982 }
7983 return Register();
7984}
7985
7986Register fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7987 if (RetVT.SimpleTy != MVT::nxv16i8)
7988 return Register();
7989 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7990 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
7991 }
7992 return Register();
7993}
7994
7995Register fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7996 if (RetVT.SimpleTy != MVT::nxv8i16)
7997 return Register();
7998 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7999 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8000 }
8001 return Register();
8002}
8003
8004Register fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8005 if (RetVT.SimpleTy != MVT::nxv4i32)
8006 return Register();
8007 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8008 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8009 }
8010 return Register();
8011}
8012
8013Register fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8014 if (RetVT.SimpleTy != MVT::nxv2i64)
8015 return Register();
8016 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8017 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8018 }
8019 return Register();
8020}
8021
8022Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8023 if (RetVT.SimpleTy != MVT::nxv2f16)
8024 return Register();
8025 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8026 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8027 }
8028 return Register();
8029}
8030
8031Register fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8032 if (RetVT.SimpleTy != MVT::nxv4f16)
8033 return Register();
8034 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8035 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8036 }
8037 return Register();
8038}
8039
8040Register fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8041 if (RetVT.SimpleTy != MVT::nxv8f16)
8042 return Register();
8043 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8044 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8045 }
8046 return Register();
8047}
8048
8049Register fastEmit_AArch64ISD_TRN1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8050 if (RetVT.SimpleTy != MVT::nxv2bf16)
8051 return Register();
8052 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8053 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8054 }
8055 return Register();
8056}
8057
8058Register fastEmit_AArch64ISD_TRN1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8059 if (RetVT.SimpleTy != MVT::nxv4bf16)
8060 return Register();
8061 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8062 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8063 }
8064 return Register();
8065}
8066
8067Register fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8068 if (RetVT.SimpleTy != MVT::nxv8bf16)
8069 return Register();
8070 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8071 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8072 }
8073 return Register();
8074}
8075
8076Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8077 if (RetVT.SimpleTy != MVT::nxv2f32)
8078 return Register();
8079 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8080 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8081 }
8082 return Register();
8083}
8084
8085Register fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8086 if (RetVT.SimpleTy != MVT::nxv4f32)
8087 return Register();
8088 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8089 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8090 }
8091 return Register();
8092}
8093
8094Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8095 if (RetVT.SimpleTy != MVT::nxv2f64)
8096 return Register();
8097 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8098 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8099 }
8100 return Register();
8101}
8102
8103Register fastEmit_AArch64ISD_TRN1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8104 switch (VT.SimpleTy) {
8105 case MVT::v8i8: return fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(RetVT, Op0, Op1);
8106 case MVT::v16i8: return fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(RetVT, Op0, Op1);
8107 case MVT::v4i16: return fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(RetVT, Op0, Op1);
8108 case MVT::v8i16: return fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(RetVT, Op0, Op1);
8109 case MVT::v2i32: return fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(RetVT, Op0, Op1);
8110 case MVT::v4i32: return fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(RetVT, Op0, Op1);
8111 case MVT::v2i64: return fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(RetVT, Op0, Op1);
8112 case MVT::v4f16: return fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(RetVT, Op0, Op1);
8113 case MVT::v8f16: return fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(RetVT, Op0, Op1);
8114 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(RetVT, Op0, Op1);
8115 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(RetVT, Op0, Op1);
8116 case MVT::v2f32: return fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(RetVT, Op0, Op1);
8117 case MVT::v4f32: return fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(RetVT, Op0, Op1);
8118 case MVT::v2f64: return fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(RetVT, Op0, Op1);
8119 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8120 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8121 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8122 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8123 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8124 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8125 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8126 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8127 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8128 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8129 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8130 case MVT::nxv2bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8131 case MVT::nxv4bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8132 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8133 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8134 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8135 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8136 default: return Register();
8137 }
8138}
8139
8140// FastEmit functions for AArch64ISD::TRN2.
8141
8142Register fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8143 if (RetVT.SimpleTy != MVT::v8i8)
8144 return Register();
8145 if ((Subtarget->isNeonAvailable())) {
8146 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8147 }
8148 return Register();
8149}
8150
8151Register fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8152 if (RetVT.SimpleTy != MVT::v16i8)
8153 return Register();
8154 if ((Subtarget->isNeonAvailable())) {
8155 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8156 }
8157 return Register();
8158}
8159
8160Register fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8161 if (RetVT.SimpleTy != MVT::v4i16)
8162 return Register();
8163 if ((Subtarget->isNeonAvailable())) {
8164 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8165 }
8166 return Register();
8167}
8168
8169Register fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8170 if (RetVT.SimpleTy != MVT::v8i16)
8171 return Register();
8172 if ((Subtarget->isNeonAvailable())) {
8173 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8174 }
8175 return Register();
8176}
8177
8178Register fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8179 if (RetVT.SimpleTy != MVT::v2i32)
8180 return Register();
8181 if ((Subtarget->isNeonAvailable())) {
8182 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8183 }
8184 return Register();
8185}
8186
8187Register fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8188 if (RetVT.SimpleTy != MVT::v4i32)
8189 return Register();
8190 if ((Subtarget->isNeonAvailable())) {
8191 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8192 }
8193 return Register();
8194}
8195
8196Register fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8197 if (RetVT.SimpleTy != MVT::v2i64)
8198 return Register();
8199 if ((Subtarget->isNeonAvailable())) {
8200 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8201 }
8202 return Register();
8203}
8204
8205Register fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8206 if (RetVT.SimpleTy != MVT::v4f16)
8207 return Register();
8208 if ((Subtarget->isNeonAvailable())) {
8209 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8210 }
8211 return Register();
8212}
8213
8214Register fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8215 if (RetVT.SimpleTy != MVT::v8f16)
8216 return Register();
8217 if ((Subtarget->isNeonAvailable())) {
8218 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8219 }
8220 return Register();
8221}
8222
8223Register fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8224 if (RetVT.SimpleTy != MVT::v4bf16)
8225 return Register();
8226 if ((Subtarget->isNeonAvailable())) {
8227 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8228 }
8229 return Register();
8230}
8231
8232Register fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8233 if (RetVT.SimpleTy != MVT::v8bf16)
8234 return Register();
8235 if ((Subtarget->isNeonAvailable())) {
8236 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8237 }
8238 return Register();
8239}
8240
8241Register fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8242 if (RetVT.SimpleTy != MVT::v2f32)
8243 return Register();
8244 if ((Subtarget->isNeonAvailable())) {
8245 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8246 }
8247 return Register();
8248}
8249
8250Register fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8251 if (RetVT.SimpleTy != MVT::v4f32)
8252 return Register();
8253 if ((Subtarget->isNeonAvailable())) {
8254 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8255 }
8256 return Register();
8257}
8258
8259Register fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8260 if (RetVT.SimpleTy != MVT::v2f64)
8261 return Register();
8262 if ((Subtarget->isNeonAvailable())) {
8263 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8264 }
8265 return Register();
8266}
8267
8268Register fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8269 if (RetVT.SimpleTy != MVT::nxv2i1)
8270 return Register();
8271 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8272 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8273 }
8274 return Register();
8275}
8276
8277Register fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8278 if (RetVT.SimpleTy != MVT::nxv4i1)
8279 return Register();
8280 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8281 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8282 }
8283 return Register();
8284}
8285
8286Register fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8287 if (RetVT.SimpleTy != MVT::nxv8i1)
8288 return Register();
8289 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8290 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8291 }
8292 return Register();
8293}
8294
8295Register fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8296 if (RetVT.SimpleTy != MVT::nxv16i1)
8297 return Register();
8298 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8299 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8300 }
8301 return Register();
8302}
8303
8304Register fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8305 if (RetVT.SimpleTy != MVT::nxv16i8)
8306 return Register();
8307 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8308 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8309 }
8310 return Register();
8311}
8312
8313Register fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8314 if (RetVT.SimpleTy != MVT::nxv8i16)
8315 return Register();
8316 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8317 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8318 }
8319 return Register();
8320}
8321
8322Register fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8323 if (RetVT.SimpleTy != MVT::nxv4i32)
8324 return Register();
8325 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8326 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8327 }
8328 return Register();
8329}
8330
8331Register fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8332 if (RetVT.SimpleTy != MVT::nxv2i64)
8333 return Register();
8334 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8335 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8336 }
8337 return Register();
8338}
8339
8340Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8341 if (RetVT.SimpleTy != MVT::nxv2f16)
8342 return Register();
8343 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8344 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8345 }
8346 return Register();
8347}
8348
8349Register fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8350 if (RetVT.SimpleTy != MVT::nxv4f16)
8351 return Register();
8352 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8353 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8354 }
8355 return Register();
8356}
8357
8358Register fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8359 if (RetVT.SimpleTy != MVT::nxv8f16)
8360 return Register();
8361 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8362 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8363 }
8364 return Register();
8365}
8366
8367Register fastEmit_AArch64ISD_TRN2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8368 if (RetVT.SimpleTy != MVT::nxv2bf16)
8369 return Register();
8370 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8371 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8372 }
8373 return Register();
8374}
8375
8376Register fastEmit_AArch64ISD_TRN2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8377 if (RetVT.SimpleTy != MVT::nxv4bf16)
8378 return Register();
8379 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8380 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8381 }
8382 return Register();
8383}
8384
8385Register fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8386 if (RetVT.SimpleTy != MVT::nxv8bf16)
8387 return Register();
8388 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8389 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8390 }
8391 return Register();
8392}
8393
8394Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8395 if (RetVT.SimpleTy != MVT::nxv2f32)
8396 return Register();
8397 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8398 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8399 }
8400 return Register();
8401}
8402
8403Register fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8404 if (RetVT.SimpleTy != MVT::nxv4f32)
8405 return Register();
8406 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8407 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8408 }
8409 return Register();
8410}
8411
8412Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8413 if (RetVT.SimpleTy != MVT::nxv2f64)
8414 return Register();
8415 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8416 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8417 }
8418 return Register();
8419}
8420
8421Register fastEmit_AArch64ISD_TRN2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8422 switch (VT.SimpleTy) {
8423 case MVT::v8i8: return fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(RetVT, Op0, Op1);
8424 case MVT::v16i8: return fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(RetVT, Op0, Op1);
8425 case MVT::v4i16: return fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(RetVT, Op0, Op1);
8426 case MVT::v8i16: return fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(RetVT, Op0, Op1);
8427 case MVT::v2i32: return fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(RetVT, Op0, Op1);
8428 case MVT::v4i32: return fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(RetVT, Op0, Op1);
8429 case MVT::v2i64: return fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(RetVT, Op0, Op1);
8430 case MVT::v4f16: return fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(RetVT, Op0, Op1);
8431 case MVT::v8f16: return fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(RetVT, Op0, Op1);
8432 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(RetVT, Op0, Op1);
8433 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(RetVT, Op0, Op1);
8434 case MVT::v2f32: return fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(RetVT, Op0, Op1);
8435 case MVT::v4f32: return fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(RetVT, Op0, Op1);
8436 case MVT::v2f64: return fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(RetVT, Op0, Op1);
8437 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8438 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8439 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8440 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8441 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8442 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8443 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8444 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8445 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8446 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8447 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8448 case MVT::nxv2bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8449 case MVT::nxv4bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8450 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8451 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8452 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8453 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8454 default: return Register();
8455 }
8456}
8457
8458// FastEmit functions for AArch64ISD::UMULL.
8459
8460Register fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8461 if (RetVT.SimpleTy != MVT::v8i16)
8462 return Register();
8463 if ((Subtarget->isNeonAvailable())) {
8464 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv8i8_v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8465 }
8466 return Register();
8467}
8468
8469Register fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8470 if (RetVT.SimpleTy != MVT::v4i32)
8471 return Register();
8472 if ((Subtarget->isNeonAvailable())) {
8473 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv4i16_v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8474 }
8475 return Register();
8476}
8477
8478Register fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8479 if (RetVT.SimpleTy != MVT::v2i64)
8480 return Register();
8481 if ((Subtarget->isNeonAvailable())) {
8482 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv2i32_v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8483 }
8484 return Register();
8485}
8486
8487Register fastEmit_AArch64ISD_UMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8488 switch (VT.SimpleTy) {
8489 case MVT::v8i8: return fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
8490 case MVT::v4i16: return fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(RetVT, Op0, Op1);
8491 case MVT::v2i32: return fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(RetVT, Op0, Op1);
8492 default: return Register();
8493 }
8494}
8495
8496// FastEmit functions for AArch64ISD::UQADD.
8497
8498Register fastEmit_AArch64ISD_UQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8499 if (RetVT.SimpleTy != MVT::f32)
8500 return Register();
8501 if ((Subtarget->isNeonAvailable())) {
8502 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8503 }
8504 return Register();
8505}
8506
8507Register fastEmit_AArch64ISD_UQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8508 if (RetVT.SimpleTy != MVT::f64)
8509 return Register();
8510 if ((Subtarget->isNeonAvailable())) {
8511 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8512 }
8513 return Register();
8514}
8515
8516Register fastEmit_AArch64ISD_UQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8517 switch (VT.SimpleTy) {
8518 case MVT::f32: return fastEmit_AArch64ISD_UQADD_MVT_f32_rr(RetVT, Op0, Op1);
8519 case MVT::f64: return fastEmit_AArch64ISD_UQADD_MVT_f64_rr(RetVT, Op0, Op1);
8520 default: return Register();
8521 }
8522}
8523
8524// FastEmit functions for AArch64ISD::UQRSHL.
8525
8526Register fastEmit_AArch64ISD_UQRSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8527 if (RetVT.SimpleTy != MVT::f32)
8528 return Register();
8529 if ((Subtarget->isNeonAvailable())) {
8530 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQRSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8531 }
8532 return Register();
8533}
8534
8535Register fastEmit_AArch64ISD_UQRSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8536 if (RetVT.SimpleTy != MVT::f64)
8537 return Register();
8538 if ((Subtarget->isNeonAvailable())) {
8539 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQRSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8540 }
8541 return Register();
8542}
8543
8544Register fastEmit_AArch64ISD_UQRSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8545 switch (VT.SimpleTy) {
8546 case MVT::f32: return fastEmit_AArch64ISD_UQRSHL_MVT_f32_rr(RetVT, Op0, Op1);
8547 case MVT::f64: return fastEmit_AArch64ISD_UQRSHL_MVT_f64_rr(RetVT, Op0, Op1);
8548 default: return Register();
8549 }
8550}
8551
8552// FastEmit functions for AArch64ISD::UQSHL.
8553
8554Register fastEmit_AArch64ISD_UQSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8555 if (RetVT.SimpleTy != MVT::f32)
8556 return Register();
8557 if ((Subtarget->isNeonAvailable())) {
8558 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8559 }
8560 return Register();
8561}
8562
8563Register fastEmit_AArch64ISD_UQSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8564 if (RetVT.SimpleTy != MVT::f64)
8565 return Register();
8566 if ((Subtarget->isNeonAvailable())) {
8567 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8568 }
8569 return Register();
8570}
8571
8572Register fastEmit_AArch64ISD_UQSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8573 switch (VT.SimpleTy) {
8574 case MVT::f32: return fastEmit_AArch64ISD_UQSHL_MVT_f32_rr(RetVT, Op0, Op1);
8575 case MVT::f64: return fastEmit_AArch64ISD_UQSHL_MVT_f64_rr(RetVT, Op0, Op1);
8576 default: return Register();
8577 }
8578}
8579
8580// FastEmit functions for AArch64ISD::UQSUB.
8581
8582Register fastEmit_AArch64ISD_UQSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8583 if (RetVT.SimpleTy != MVT::f32)
8584 return Register();
8585 if ((Subtarget->isNeonAvailable())) {
8586 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8587 }
8588 return Register();
8589}
8590
8591Register fastEmit_AArch64ISD_UQSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8592 if (RetVT.SimpleTy != MVT::f64)
8593 return Register();
8594 if ((Subtarget->isNeonAvailable())) {
8595 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8596 }
8597 return Register();
8598}
8599
8600Register fastEmit_AArch64ISD_UQSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8601 switch (VT.SimpleTy) {
8602 case MVT::f32: return fastEmit_AArch64ISD_UQSUB_MVT_f32_rr(RetVT, Op0, Op1);
8603 case MVT::f64: return fastEmit_AArch64ISD_UQSUB_MVT_f64_rr(RetVT, Op0, Op1);
8604 default: return Register();
8605 }
8606}
8607
8608// FastEmit functions for AArch64ISD::USQADD.
8609
8610Register fastEmit_AArch64ISD_USQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8611 if (RetVT.SimpleTy != MVT::f32)
8612 return Register();
8613 if ((Subtarget->isNeonAvailable())) {
8614 return fastEmitInst_rr(MachineInstOpcode: AArch64::USQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8615 }
8616 return Register();
8617}
8618
8619Register fastEmit_AArch64ISD_USQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8620 if (RetVT.SimpleTy != MVT::f64)
8621 return Register();
8622 if ((Subtarget->isNeonAvailable())) {
8623 return fastEmitInst_rr(MachineInstOpcode: AArch64::USQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8624 }
8625 return Register();
8626}
8627
8628Register fastEmit_AArch64ISD_USQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8629 switch (VT.SimpleTy) {
8630 case MVT::f32: return fastEmit_AArch64ISD_USQADD_MVT_f32_rr(RetVT, Op0, Op1);
8631 case MVT::f64: return fastEmit_AArch64ISD_USQADD_MVT_f64_rr(RetVT, Op0, Op1);
8632 default: return Register();
8633 }
8634}
8635
8636// FastEmit functions for AArch64ISD::UZP1.
8637
8638Register fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8639 if (RetVT.SimpleTy != MVT::v8i8)
8640 return Register();
8641 if ((Subtarget->isNeonAvailable())) {
8642 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8643 }
8644 return Register();
8645}
8646
8647Register fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8648 if (RetVT.SimpleTy != MVT::v16i8)
8649 return Register();
8650 if ((Subtarget->isNeonAvailable())) {
8651 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8652 }
8653 return Register();
8654}
8655
8656Register fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8657 if (RetVT.SimpleTy != MVT::v4i16)
8658 return Register();
8659 if ((Subtarget->isNeonAvailable())) {
8660 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8661 }
8662 return Register();
8663}
8664
8665Register fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8666 if (RetVT.SimpleTy != MVT::v8i16)
8667 return Register();
8668 if ((Subtarget->isNeonAvailable())) {
8669 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8670 }
8671 return Register();
8672}
8673
8674Register fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8675 if (RetVT.SimpleTy != MVT::v2i32)
8676 return Register();
8677 if ((Subtarget->isNeonAvailable())) {
8678 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8679 }
8680 return Register();
8681}
8682
8683Register fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8684 if (RetVT.SimpleTy != MVT::v4i32)
8685 return Register();
8686 if ((Subtarget->isNeonAvailable())) {
8687 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8688 }
8689 return Register();
8690}
8691
8692Register fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8693 if (RetVT.SimpleTy != MVT::v2i64)
8694 return Register();
8695 if ((Subtarget->isNeonAvailable())) {
8696 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8697 }
8698 return Register();
8699}
8700
8701Register fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8702 if (RetVT.SimpleTy != MVT::v4f16)
8703 return Register();
8704 if ((Subtarget->isNeonAvailable())) {
8705 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8706 }
8707 return Register();
8708}
8709
8710Register fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8711 if (RetVT.SimpleTy != MVT::v8f16)
8712 return Register();
8713 if ((Subtarget->isNeonAvailable())) {
8714 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8715 }
8716 return Register();
8717}
8718
8719Register fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8720 if (RetVT.SimpleTy != MVT::v4bf16)
8721 return Register();
8722 if ((Subtarget->isNeonAvailable())) {
8723 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8724 }
8725 return Register();
8726}
8727
8728Register fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8729 if (RetVT.SimpleTy != MVT::v8bf16)
8730 return Register();
8731 if ((Subtarget->isNeonAvailable())) {
8732 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8733 }
8734 return Register();
8735}
8736
8737Register fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8738 if (RetVT.SimpleTy != MVT::v2f32)
8739 return Register();
8740 if ((Subtarget->isNeonAvailable())) {
8741 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8742 }
8743 return Register();
8744}
8745
8746Register fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8747 if (RetVT.SimpleTy != MVT::v4f32)
8748 return Register();
8749 if ((Subtarget->isNeonAvailable())) {
8750 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8751 }
8752 return Register();
8753}
8754
8755Register fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8756 if (RetVT.SimpleTy != MVT::v2f64)
8757 return Register();
8758 if ((Subtarget->isNeonAvailable())) {
8759 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8760 }
8761 return Register();
8762}
8763
8764Register fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8765 if (RetVT.SimpleTy != MVT::nxv2i1)
8766 return Register();
8767 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8768 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8769 }
8770 return Register();
8771}
8772
8773Register fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8774 if (RetVT.SimpleTy != MVT::nxv4i1)
8775 return Register();
8776 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8777 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8778 }
8779 return Register();
8780}
8781
8782Register fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8783 if (RetVT.SimpleTy != MVT::nxv8i1)
8784 return Register();
8785 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8786 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8787 }
8788 return Register();
8789}
8790
8791Register fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8792 if (RetVT.SimpleTy != MVT::nxv16i1)
8793 return Register();
8794 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8795 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8796 }
8797 return Register();
8798}
8799
8800Register fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8801 if (RetVT.SimpleTy != MVT::nxv16i8)
8802 return Register();
8803 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8804 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8805 }
8806 return Register();
8807}
8808
8809Register fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8810 if (RetVT.SimpleTy != MVT::nxv8i16)
8811 return Register();
8812 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8813 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8814 }
8815 return Register();
8816}
8817
8818Register fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8819 if (RetVT.SimpleTy != MVT::nxv4i32)
8820 return Register();
8821 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8822 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8823 }
8824 return Register();
8825}
8826
8827Register fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8828 if (RetVT.SimpleTy != MVT::nxv2i64)
8829 return Register();
8830 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8831 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8832 }
8833 return Register();
8834}
8835
8836Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8837 if (RetVT.SimpleTy != MVT::nxv2f16)
8838 return Register();
8839 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8840 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8841 }
8842 return Register();
8843}
8844
8845Register fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8846 if (RetVT.SimpleTy != MVT::nxv4f16)
8847 return Register();
8848 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8849 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8850 }
8851 return Register();
8852}
8853
8854Register fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8855 if (RetVT.SimpleTy != MVT::nxv8f16)
8856 return Register();
8857 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8858 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8859 }
8860 return Register();
8861}
8862
8863Register fastEmit_AArch64ISD_UZP1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8864 if (RetVT.SimpleTy != MVT::nxv2bf16)
8865 return Register();
8866 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8867 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8868 }
8869 return Register();
8870}
8871
8872Register fastEmit_AArch64ISD_UZP1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8873 if (RetVT.SimpleTy != MVT::nxv4bf16)
8874 return Register();
8875 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8876 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8877 }
8878 return Register();
8879}
8880
8881Register fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8882 if (RetVT.SimpleTy != MVT::nxv8bf16)
8883 return Register();
8884 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8885 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8886 }
8887 return Register();
8888}
8889
8890Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8891 if (RetVT.SimpleTy != MVT::nxv2f32)
8892 return Register();
8893 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8894 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8895 }
8896 return Register();
8897}
8898
8899Register fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8900 if (RetVT.SimpleTy != MVT::nxv4f32)
8901 return Register();
8902 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8903 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8904 }
8905 return Register();
8906}
8907
8908Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8909 if (RetVT.SimpleTy != MVT::nxv2f64)
8910 return Register();
8911 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8912 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8913 }
8914 return Register();
8915}
8916
8917Register fastEmit_AArch64ISD_UZP1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8918 switch (VT.SimpleTy) {
8919 case MVT::v8i8: return fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(RetVT, Op0, Op1);
8920 case MVT::v16i8: return fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(RetVT, Op0, Op1);
8921 case MVT::v4i16: return fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(RetVT, Op0, Op1);
8922 case MVT::v8i16: return fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(RetVT, Op0, Op1);
8923 case MVT::v2i32: return fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(RetVT, Op0, Op1);
8924 case MVT::v4i32: return fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(RetVT, Op0, Op1);
8925 case MVT::v2i64: return fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(RetVT, Op0, Op1);
8926 case MVT::v4f16: return fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(RetVT, Op0, Op1);
8927 case MVT::v8f16: return fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(RetVT, Op0, Op1);
8928 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(RetVT, Op0, Op1);
8929 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(RetVT, Op0, Op1);
8930 case MVT::v2f32: return fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(RetVT, Op0, Op1);
8931 case MVT::v4f32: return fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(RetVT, Op0, Op1);
8932 case MVT::v2f64: return fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(RetVT, Op0, Op1);
8933 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8934 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8935 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8936 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8937 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8938 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8939 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8940 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8941 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8942 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8943 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8944 case MVT::nxv2bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8945 case MVT::nxv4bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8946 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8947 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8948 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8949 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8950 default: return Register();
8951 }
8952}
8953
8954// FastEmit functions for AArch64ISD::UZP2.
8955
8956Register fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8957 if (RetVT.SimpleTy != MVT::v8i8)
8958 return Register();
8959 if ((Subtarget->isNeonAvailable())) {
8960 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8961 }
8962 return Register();
8963}
8964
8965Register fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8966 if (RetVT.SimpleTy != MVT::v16i8)
8967 return Register();
8968 if ((Subtarget->isNeonAvailable())) {
8969 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8970 }
8971 return Register();
8972}
8973
8974Register fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8975 if (RetVT.SimpleTy != MVT::v4i16)
8976 return Register();
8977 if ((Subtarget->isNeonAvailable())) {
8978 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8979 }
8980 return Register();
8981}
8982
8983Register fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8984 if (RetVT.SimpleTy != MVT::v8i16)
8985 return Register();
8986 if ((Subtarget->isNeonAvailable())) {
8987 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8988 }
8989 return Register();
8990}
8991
8992Register fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8993 if (RetVT.SimpleTy != MVT::v2i32)
8994 return Register();
8995 if ((Subtarget->isNeonAvailable())) {
8996 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8997 }
8998 return Register();
8999}
9000
9001Register fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9002 if (RetVT.SimpleTy != MVT::v4i32)
9003 return Register();
9004 if ((Subtarget->isNeonAvailable())) {
9005 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9006 }
9007 return Register();
9008}
9009
9010Register fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9011 if (RetVT.SimpleTy != MVT::v2i64)
9012 return Register();
9013 if ((Subtarget->isNeonAvailable())) {
9014 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9015 }
9016 return Register();
9017}
9018
9019Register fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9020 if (RetVT.SimpleTy != MVT::v4f16)
9021 return Register();
9022 if ((Subtarget->isNeonAvailable())) {
9023 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9024 }
9025 return Register();
9026}
9027
9028Register fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9029 if (RetVT.SimpleTy != MVT::v8f16)
9030 return Register();
9031 if ((Subtarget->isNeonAvailable())) {
9032 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9033 }
9034 return Register();
9035}
9036
9037Register fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9038 if (RetVT.SimpleTy != MVT::v4bf16)
9039 return Register();
9040 if ((Subtarget->isNeonAvailable())) {
9041 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9042 }
9043 return Register();
9044}
9045
9046Register fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9047 if (RetVT.SimpleTy != MVT::v8bf16)
9048 return Register();
9049 if ((Subtarget->isNeonAvailable())) {
9050 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9051 }
9052 return Register();
9053}
9054
9055Register fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9056 if (RetVT.SimpleTy != MVT::v2f32)
9057 return Register();
9058 if ((Subtarget->isNeonAvailable())) {
9059 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9060 }
9061 return Register();
9062}
9063
9064Register fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9065 if (RetVT.SimpleTy != MVT::v4f32)
9066 return Register();
9067 if ((Subtarget->isNeonAvailable())) {
9068 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9069 }
9070 return Register();
9071}
9072
9073Register fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9074 if (RetVT.SimpleTy != MVT::v2f64)
9075 return Register();
9076 if ((Subtarget->isNeonAvailable())) {
9077 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9078 }
9079 return Register();
9080}
9081
9082Register fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9083 if (RetVT.SimpleTy != MVT::nxv2i1)
9084 return Register();
9085 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9086 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9087 }
9088 return Register();
9089}
9090
9091Register fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9092 if (RetVT.SimpleTy != MVT::nxv4i1)
9093 return Register();
9094 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9095 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9096 }
9097 return Register();
9098}
9099
9100Register fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9101 if (RetVT.SimpleTy != MVT::nxv8i1)
9102 return Register();
9103 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9104 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9105 }
9106 return Register();
9107}
9108
9109Register fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9110 if (RetVT.SimpleTy != MVT::nxv16i1)
9111 return Register();
9112 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9113 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9114 }
9115 return Register();
9116}
9117
9118Register fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9119 if (RetVT.SimpleTy != MVT::nxv16i8)
9120 return Register();
9121 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9122 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9123 }
9124 return Register();
9125}
9126
9127Register fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9128 if (RetVT.SimpleTy != MVT::nxv8i16)
9129 return Register();
9130 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9131 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9132 }
9133 return Register();
9134}
9135
9136Register fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9137 if (RetVT.SimpleTy != MVT::nxv4i32)
9138 return Register();
9139 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9140 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9141 }
9142 return Register();
9143}
9144
9145Register fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9146 if (RetVT.SimpleTy != MVT::nxv2i64)
9147 return Register();
9148 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9149 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9150 }
9151 return Register();
9152}
9153
9154Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9155 if (RetVT.SimpleTy != MVT::nxv2f16)
9156 return Register();
9157 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9158 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9159 }
9160 return Register();
9161}
9162
9163Register fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9164 if (RetVT.SimpleTy != MVT::nxv4f16)
9165 return Register();
9166 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9167 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9168 }
9169 return Register();
9170}
9171
9172Register fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9173 if (RetVT.SimpleTy != MVT::nxv8f16)
9174 return Register();
9175 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9176 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9177 }
9178 return Register();
9179}
9180
9181Register fastEmit_AArch64ISD_UZP2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9182 if (RetVT.SimpleTy != MVT::nxv2bf16)
9183 return Register();
9184 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9185 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9186 }
9187 return Register();
9188}
9189
9190Register fastEmit_AArch64ISD_UZP2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9191 if (RetVT.SimpleTy != MVT::nxv4bf16)
9192 return Register();
9193 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9194 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9195 }
9196 return Register();
9197}
9198
9199Register fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9200 if (RetVT.SimpleTy != MVT::nxv8bf16)
9201 return Register();
9202 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9203 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9204 }
9205 return Register();
9206}
9207
9208Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9209 if (RetVT.SimpleTy != MVT::nxv2f32)
9210 return Register();
9211 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9212 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9213 }
9214 return Register();
9215}
9216
9217Register fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9218 if (RetVT.SimpleTy != MVT::nxv4f32)
9219 return Register();
9220 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9221 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9222 }
9223 return Register();
9224}
9225
9226Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9227 if (RetVT.SimpleTy != MVT::nxv2f64)
9228 return Register();
9229 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9230 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9231 }
9232 return Register();
9233}
9234
9235Register fastEmit_AArch64ISD_UZP2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9236 switch (VT.SimpleTy) {
9237 case MVT::v8i8: return fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(RetVT, Op0, Op1);
9238 case MVT::v16i8: return fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(RetVT, Op0, Op1);
9239 case MVT::v4i16: return fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(RetVT, Op0, Op1);
9240 case MVT::v8i16: return fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(RetVT, Op0, Op1);
9241 case MVT::v2i32: return fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(RetVT, Op0, Op1);
9242 case MVT::v4i32: return fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(RetVT, Op0, Op1);
9243 case MVT::v2i64: return fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(RetVT, Op0, Op1);
9244 case MVT::v4f16: return fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(RetVT, Op0, Op1);
9245 case MVT::v8f16: return fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(RetVT, Op0, Op1);
9246 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(RetVT, Op0, Op1);
9247 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(RetVT, Op0, Op1);
9248 case MVT::v2f32: return fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(RetVT, Op0, Op1);
9249 case MVT::v4f32: return fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(RetVT, Op0, Op1);
9250 case MVT::v2f64: return fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(RetVT, Op0, Op1);
9251 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9252 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9253 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9254 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9255 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9256 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9257 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9258 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9259 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9260 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9261 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9262 case MVT::nxv2bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9263 case MVT::nxv4bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9264 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9265 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9266 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9267 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9268 default: return Register();
9269 }
9270}
9271
9272// FastEmit functions for AArch64ISD::ZIP1.
9273
9274Register fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9275 if (RetVT.SimpleTy != MVT::v8i8)
9276 return Register();
9277 if ((Subtarget->isNeonAvailable())) {
9278 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9279 }
9280 return Register();
9281}
9282
9283Register fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9284 if (RetVT.SimpleTy != MVT::v16i8)
9285 return Register();
9286 if ((Subtarget->isNeonAvailable())) {
9287 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9288 }
9289 return Register();
9290}
9291
9292Register fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9293 if (RetVT.SimpleTy != MVT::v4i16)
9294 return Register();
9295 if ((Subtarget->isNeonAvailable())) {
9296 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9297 }
9298 return Register();
9299}
9300
9301Register fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9302 if (RetVT.SimpleTy != MVT::v8i16)
9303 return Register();
9304 if ((Subtarget->isNeonAvailable())) {
9305 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9306 }
9307 return Register();
9308}
9309
9310Register fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9311 if (RetVT.SimpleTy != MVT::v2i32)
9312 return Register();
9313 if ((Subtarget->isNeonAvailable())) {
9314 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9315 }
9316 return Register();
9317}
9318
9319Register fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9320 if (RetVT.SimpleTy != MVT::v4i32)
9321 return Register();
9322 if ((Subtarget->isNeonAvailable())) {
9323 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9324 }
9325 return Register();
9326}
9327
9328Register fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9329 if (RetVT.SimpleTy != MVT::v2i64)
9330 return Register();
9331 if ((Subtarget->isNeonAvailable())) {
9332 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9333 }
9334 return Register();
9335}
9336
9337Register fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9338 if (RetVT.SimpleTy != MVT::v4f16)
9339 return Register();
9340 if ((Subtarget->isNeonAvailable())) {
9341 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9342 }
9343 return Register();
9344}
9345
9346Register fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9347 if (RetVT.SimpleTy != MVT::v8f16)
9348 return Register();
9349 if ((Subtarget->isNeonAvailable())) {
9350 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9351 }
9352 return Register();
9353}
9354
9355Register fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9356 if (RetVT.SimpleTy != MVT::v4bf16)
9357 return Register();
9358 if ((Subtarget->isNeonAvailable())) {
9359 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9360 }
9361 return Register();
9362}
9363
9364Register fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9365 if (RetVT.SimpleTy != MVT::v8bf16)
9366 return Register();
9367 if ((Subtarget->isNeonAvailable())) {
9368 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9369 }
9370 return Register();
9371}
9372
9373Register fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9374 if (RetVT.SimpleTy != MVT::v2f32)
9375 return Register();
9376 if ((Subtarget->isNeonAvailable())) {
9377 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9378 }
9379 return Register();
9380}
9381
9382Register fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9383 if (RetVT.SimpleTy != MVT::v4f32)
9384 return Register();
9385 if ((Subtarget->isNeonAvailable())) {
9386 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9387 }
9388 return Register();
9389}
9390
9391Register fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9392 if (RetVT.SimpleTy != MVT::v2f64)
9393 return Register();
9394 if ((Subtarget->isNeonAvailable())) {
9395 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9396 }
9397 return Register();
9398}
9399
9400Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9401 if (RetVT.SimpleTy != MVT::nxv2i1)
9402 return Register();
9403 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9404 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9405 }
9406 return Register();
9407}
9408
9409Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9410 if (RetVT.SimpleTy != MVT::nxv4i1)
9411 return Register();
9412 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9413 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9414 }
9415 return Register();
9416}
9417
9418Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9419 if (RetVT.SimpleTy != MVT::nxv8i1)
9420 return Register();
9421 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9422 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9423 }
9424 return Register();
9425}
9426
9427Register fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9428 if (RetVT.SimpleTy != MVT::nxv16i1)
9429 return Register();
9430 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9431 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9432 }
9433 return Register();
9434}
9435
9436Register fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9437 if (RetVT.SimpleTy != MVT::nxv16i8)
9438 return Register();
9439 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9440 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9441 }
9442 return Register();
9443}
9444
9445Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9446 if (RetVT.SimpleTy != MVT::nxv8i16)
9447 return Register();
9448 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9449 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9450 }
9451 return Register();
9452}
9453
9454Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9455 if (RetVT.SimpleTy != MVT::nxv4i32)
9456 return Register();
9457 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9458 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9459 }
9460 return Register();
9461}
9462
9463Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9464 if (RetVT.SimpleTy != MVT::nxv2i64)
9465 return Register();
9466 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9467 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9468 }
9469 return Register();
9470}
9471
9472Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9473 if (RetVT.SimpleTy != MVT::nxv2f16)
9474 return Register();
9475 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9476 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9477 }
9478 return Register();
9479}
9480
9481Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9482 if (RetVT.SimpleTy != MVT::nxv4f16)
9483 return Register();
9484 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9485 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9486 }
9487 return Register();
9488}
9489
9490Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9491 if (RetVT.SimpleTy != MVT::nxv8f16)
9492 return Register();
9493 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9494 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9495 }
9496 return Register();
9497}
9498
9499Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9500 if (RetVT.SimpleTy != MVT::nxv2bf16)
9501 return Register();
9502 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9503 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9504 }
9505 return Register();
9506}
9507
9508Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9509 if (RetVT.SimpleTy != MVT::nxv4bf16)
9510 return Register();
9511 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9512 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9513 }
9514 return Register();
9515}
9516
9517Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9518 if (RetVT.SimpleTy != MVT::nxv8bf16)
9519 return Register();
9520 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9521 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9522 }
9523 return Register();
9524}
9525
9526Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9527 if (RetVT.SimpleTy != MVT::nxv2f32)
9528 return Register();
9529 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9530 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9531 }
9532 return Register();
9533}
9534
9535Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9536 if (RetVT.SimpleTy != MVT::nxv4f32)
9537 return Register();
9538 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9539 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9540 }
9541 return Register();
9542}
9543
9544Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9545 if (RetVT.SimpleTy != MVT::nxv2f64)
9546 return Register();
9547 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9548 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9549 }
9550 return Register();
9551}
9552
9553Register fastEmit_AArch64ISD_ZIP1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9554 switch (VT.SimpleTy) {
9555 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(RetVT, Op0, Op1);
9556 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(RetVT, Op0, Op1);
9557 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(RetVT, Op0, Op1);
9558 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(RetVT, Op0, Op1);
9559 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(RetVT, Op0, Op1);
9560 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(RetVT, Op0, Op1);
9561 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(RetVT, Op0, Op1);
9562 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(RetVT, Op0, Op1);
9563 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(RetVT, Op0, Op1);
9564 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(RetVT, Op0, Op1);
9565 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(RetVT, Op0, Op1);
9566 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(RetVT, Op0, Op1);
9567 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(RetVT, Op0, Op1);
9568 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(RetVT, Op0, Op1);
9569 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9570 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9571 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9572 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9573 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9574 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9575 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9576 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9577 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9578 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9579 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9580 case MVT::nxv2bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9581 case MVT::nxv4bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9582 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9583 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9584 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9585 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9586 default: return Register();
9587 }
9588}
9589
9590// FastEmit functions for AArch64ISD::ZIP2.
9591
9592Register fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9593 if (RetVT.SimpleTy != MVT::v8i8)
9594 return Register();
9595 if ((Subtarget->isNeonAvailable())) {
9596 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9597 }
9598 return Register();
9599}
9600
9601Register fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9602 if (RetVT.SimpleTy != MVT::v16i8)
9603 return Register();
9604 if ((Subtarget->isNeonAvailable())) {
9605 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9606 }
9607 return Register();
9608}
9609
9610Register fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9611 if (RetVT.SimpleTy != MVT::v4i16)
9612 return Register();
9613 if ((Subtarget->isNeonAvailable())) {
9614 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9615 }
9616 return Register();
9617}
9618
9619Register fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9620 if (RetVT.SimpleTy != MVT::v8i16)
9621 return Register();
9622 if ((Subtarget->isNeonAvailable())) {
9623 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9624 }
9625 return Register();
9626}
9627
9628Register fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9629 if (RetVT.SimpleTy != MVT::v2i32)
9630 return Register();
9631 if ((Subtarget->isNeonAvailable())) {
9632 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9633 }
9634 return Register();
9635}
9636
9637Register fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9638 if (RetVT.SimpleTy != MVT::v4i32)
9639 return Register();
9640 if ((Subtarget->isNeonAvailable())) {
9641 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9642 }
9643 return Register();
9644}
9645
9646Register fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9647 if (RetVT.SimpleTy != MVT::v2i64)
9648 return Register();
9649 if ((Subtarget->isNeonAvailable())) {
9650 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9651 }
9652 return Register();
9653}
9654
9655Register fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9656 if (RetVT.SimpleTy != MVT::v4f16)
9657 return Register();
9658 if ((Subtarget->isNeonAvailable())) {
9659 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9660 }
9661 return Register();
9662}
9663
9664Register fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9665 if (RetVT.SimpleTy != MVT::v8f16)
9666 return Register();
9667 if ((Subtarget->isNeonAvailable())) {
9668 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9669 }
9670 return Register();
9671}
9672
9673Register fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9674 if (RetVT.SimpleTy != MVT::v4bf16)
9675 return Register();
9676 if ((Subtarget->isNeonAvailable())) {
9677 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9678 }
9679 return Register();
9680}
9681
9682Register fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9683 if (RetVT.SimpleTy != MVT::v8bf16)
9684 return Register();
9685 if ((Subtarget->isNeonAvailable())) {
9686 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9687 }
9688 return Register();
9689}
9690
9691Register fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9692 if (RetVT.SimpleTy != MVT::v2f32)
9693 return Register();
9694 if ((Subtarget->isNeonAvailable())) {
9695 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9696 }
9697 return Register();
9698}
9699
9700Register fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9701 if (RetVT.SimpleTy != MVT::v4f32)
9702 return Register();
9703 if ((Subtarget->isNeonAvailable())) {
9704 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9705 }
9706 return Register();
9707}
9708
9709Register fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9710 if (RetVT.SimpleTy != MVT::v2f64)
9711 return Register();
9712 if ((Subtarget->isNeonAvailable())) {
9713 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9714 }
9715 return Register();
9716}
9717
9718Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9719 if (RetVT.SimpleTy != MVT::nxv2i1)
9720 return Register();
9721 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9722 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9723 }
9724 return Register();
9725}
9726
9727Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9728 if (RetVT.SimpleTy != MVT::nxv4i1)
9729 return Register();
9730 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9731 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9732 }
9733 return Register();
9734}
9735
9736Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9737 if (RetVT.SimpleTy != MVT::nxv8i1)
9738 return Register();
9739 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9740 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9741 }
9742 return Register();
9743}
9744
9745Register fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9746 if (RetVT.SimpleTy != MVT::nxv16i1)
9747 return Register();
9748 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9749 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9750 }
9751 return Register();
9752}
9753
9754Register fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9755 if (RetVT.SimpleTy != MVT::nxv16i8)
9756 return Register();
9757 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9758 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9759 }
9760 return Register();
9761}
9762
9763Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9764 if (RetVT.SimpleTy != MVT::nxv8i16)
9765 return Register();
9766 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9767 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9768 }
9769 return Register();
9770}
9771
9772Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9773 if (RetVT.SimpleTy != MVT::nxv4i32)
9774 return Register();
9775 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9776 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9777 }
9778 return Register();
9779}
9780
9781Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9782 if (RetVT.SimpleTy != MVT::nxv2i64)
9783 return Register();
9784 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9785 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9786 }
9787 return Register();
9788}
9789
9790Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9791 if (RetVT.SimpleTy != MVT::nxv2f16)
9792 return Register();
9793 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9794 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9795 }
9796 return Register();
9797}
9798
9799Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9800 if (RetVT.SimpleTy != MVT::nxv4f16)
9801 return Register();
9802 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9803 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9804 }
9805 return Register();
9806}
9807
9808Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9809 if (RetVT.SimpleTy != MVT::nxv8f16)
9810 return Register();
9811 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9812 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9813 }
9814 return Register();
9815}
9816
9817Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9818 if (RetVT.SimpleTy != MVT::nxv2bf16)
9819 return Register();
9820 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9821 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9822 }
9823 return Register();
9824}
9825
9826Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9827 if (RetVT.SimpleTy != MVT::nxv4bf16)
9828 return Register();
9829 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9830 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9831 }
9832 return Register();
9833}
9834
9835Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9836 if (RetVT.SimpleTy != MVT::nxv8bf16)
9837 return Register();
9838 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9839 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9840 }
9841 return Register();
9842}
9843
9844Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9845 if (RetVT.SimpleTy != MVT::nxv2f32)
9846 return Register();
9847 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9848 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9849 }
9850 return Register();
9851}
9852
9853Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9854 if (RetVT.SimpleTy != MVT::nxv4f32)
9855 return Register();
9856 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9857 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9858 }
9859 return Register();
9860}
9861
9862Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9863 if (RetVT.SimpleTy != MVT::nxv2f64)
9864 return Register();
9865 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9866 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9867 }
9868 return Register();
9869}
9870
9871Register fastEmit_AArch64ISD_ZIP2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9872 switch (VT.SimpleTy) {
9873 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(RetVT, Op0, Op1);
9874 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(RetVT, Op0, Op1);
9875 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(RetVT, Op0, Op1);
9876 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(RetVT, Op0, Op1);
9877 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(RetVT, Op0, Op1);
9878 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(RetVT, Op0, Op1);
9879 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(RetVT, Op0, Op1);
9880 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(RetVT, Op0, Op1);
9881 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(RetVT, Op0, Op1);
9882 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(RetVT, Op0, Op1);
9883 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(RetVT, Op0, Op1);
9884 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(RetVT, Op0, Op1);
9885 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(RetVT, Op0, Op1);
9886 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(RetVT, Op0, Op1);
9887 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9888 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9889 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9890 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9891 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9892 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9893 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9894 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9895 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9896 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9897 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9898 case MVT::nxv2bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9899 case MVT::nxv4bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9900 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9901 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9902 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9903 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9904 default: return Register();
9905 }
9906}
9907
9908// FastEmit functions for ISD::ABDS.
9909
9910Register fastEmit_ISD_ABDS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9911 if (RetVT.SimpleTy != MVT::v8i8)
9912 return Register();
9913 if ((Subtarget->isNeonAvailable())) {
9914 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9915 }
9916 return Register();
9917}
9918
9919Register fastEmit_ISD_ABDS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9920 if (RetVT.SimpleTy != MVT::v16i8)
9921 return Register();
9922 if ((Subtarget->isNeonAvailable())) {
9923 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9924 }
9925 return Register();
9926}
9927
9928Register fastEmit_ISD_ABDS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9929 if (RetVT.SimpleTy != MVT::v4i16)
9930 return Register();
9931 if ((Subtarget->isNeonAvailable())) {
9932 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9933 }
9934 return Register();
9935}
9936
9937Register fastEmit_ISD_ABDS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9938 if (RetVT.SimpleTy != MVT::v8i16)
9939 return Register();
9940 if ((Subtarget->isNeonAvailable())) {
9941 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9942 }
9943 return Register();
9944}
9945
9946Register fastEmit_ISD_ABDS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9947 if (RetVT.SimpleTy != MVT::v2i32)
9948 return Register();
9949 if ((Subtarget->isNeonAvailable())) {
9950 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9951 }
9952 return Register();
9953}
9954
9955Register fastEmit_ISD_ABDS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9956 if (RetVT.SimpleTy != MVT::v4i32)
9957 return Register();
9958 if ((Subtarget->isNeonAvailable())) {
9959 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9960 }
9961 return Register();
9962}
9963
9964Register fastEmit_ISD_ABDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9965 switch (VT.SimpleTy) {
9966 case MVT::v8i8: return fastEmit_ISD_ABDS_MVT_v8i8_rr(RetVT, Op0, Op1);
9967 case MVT::v16i8: return fastEmit_ISD_ABDS_MVT_v16i8_rr(RetVT, Op0, Op1);
9968 case MVT::v4i16: return fastEmit_ISD_ABDS_MVT_v4i16_rr(RetVT, Op0, Op1);
9969 case MVT::v8i16: return fastEmit_ISD_ABDS_MVT_v8i16_rr(RetVT, Op0, Op1);
9970 case MVT::v2i32: return fastEmit_ISD_ABDS_MVT_v2i32_rr(RetVT, Op0, Op1);
9971 case MVT::v4i32: return fastEmit_ISD_ABDS_MVT_v4i32_rr(RetVT, Op0, Op1);
9972 default: return Register();
9973 }
9974}
9975
9976// FastEmit functions for ISD::ABDU.
9977
9978Register fastEmit_ISD_ABDU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9979 if (RetVT.SimpleTy != MVT::v8i8)
9980 return Register();
9981 if ((Subtarget->isNeonAvailable())) {
9982 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9983 }
9984 return Register();
9985}
9986
9987Register fastEmit_ISD_ABDU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9988 if (RetVT.SimpleTy != MVT::v16i8)
9989 return Register();
9990 if ((Subtarget->isNeonAvailable())) {
9991 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9992 }
9993 return Register();
9994}
9995
9996Register fastEmit_ISD_ABDU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9997 if (RetVT.SimpleTy != MVT::v4i16)
9998 return Register();
9999 if ((Subtarget->isNeonAvailable())) {
10000 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10001 }
10002 return Register();
10003}
10004
10005Register fastEmit_ISD_ABDU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10006 if (RetVT.SimpleTy != MVT::v8i16)
10007 return Register();
10008 if ((Subtarget->isNeonAvailable())) {
10009 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10010 }
10011 return Register();
10012}
10013
10014Register fastEmit_ISD_ABDU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10015 if (RetVT.SimpleTy != MVT::v2i32)
10016 return Register();
10017 if ((Subtarget->isNeonAvailable())) {
10018 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10019 }
10020 return Register();
10021}
10022
10023Register fastEmit_ISD_ABDU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10024 if (RetVT.SimpleTy != MVT::v4i32)
10025 return Register();
10026 if ((Subtarget->isNeonAvailable())) {
10027 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10028 }
10029 return Register();
10030}
10031
10032Register fastEmit_ISD_ABDU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10033 switch (VT.SimpleTy) {
10034 case MVT::v8i8: return fastEmit_ISD_ABDU_MVT_v8i8_rr(RetVT, Op0, Op1);
10035 case MVT::v16i8: return fastEmit_ISD_ABDU_MVT_v16i8_rr(RetVT, Op0, Op1);
10036 case MVT::v4i16: return fastEmit_ISD_ABDU_MVT_v4i16_rr(RetVT, Op0, Op1);
10037 case MVT::v8i16: return fastEmit_ISD_ABDU_MVT_v8i16_rr(RetVT, Op0, Op1);
10038 case MVT::v2i32: return fastEmit_ISD_ABDU_MVT_v2i32_rr(RetVT, Op0, Op1);
10039 case MVT::v4i32: return fastEmit_ISD_ABDU_MVT_v4i32_rr(RetVT, Op0, Op1);
10040 default: return Register();
10041 }
10042}
10043
10044// FastEmit functions for ISD::ADD.
10045
10046Register fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10047 if (RetVT.SimpleTy != MVT::i32)
10048 return Register();
10049 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
10050}
10051
10052Register fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10053 if (RetVT.SimpleTy != MVT::i64)
10054 return Register();
10055 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
10056}
10057
10058Register fastEmit_ISD_ADD_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10059 if (RetVT.SimpleTy != MVT::v8i8)
10060 return Register();
10061 if ((Subtarget->isNeonAvailable())) {
10062 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10063 }
10064 return Register();
10065}
10066
10067Register fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10068 if (RetVT.SimpleTy != MVT::v16i8)
10069 return Register();
10070 if ((Subtarget->isNeonAvailable())) {
10071 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10072 }
10073 return Register();
10074}
10075
10076Register fastEmit_ISD_ADD_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10077 if (RetVT.SimpleTy != MVT::v4i16)
10078 return Register();
10079 if ((Subtarget->isNeonAvailable())) {
10080 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10081 }
10082 return Register();
10083}
10084
10085Register fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10086 if (RetVT.SimpleTy != MVT::v8i16)
10087 return Register();
10088 if ((Subtarget->isNeonAvailable())) {
10089 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10090 }
10091 return Register();
10092}
10093
10094Register fastEmit_ISD_ADD_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10095 if (RetVT.SimpleTy != MVT::v2i32)
10096 return Register();
10097 if ((Subtarget->isNeonAvailable())) {
10098 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10099 }
10100 return Register();
10101}
10102
10103Register fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10104 if (RetVT.SimpleTy != MVT::v4i32)
10105 return Register();
10106 if ((Subtarget->isNeonAvailable())) {
10107 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10108 }
10109 return Register();
10110}
10111
10112Register fastEmit_ISD_ADD_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
10113 if (RetVT.SimpleTy != MVT::v1i64)
10114 return Register();
10115 if ((Subtarget->isNeonAvailable())) {
10116 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
10117 }
10118 return Register();
10119}
10120
10121Register fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10122 if (RetVT.SimpleTy != MVT::v2i64)
10123 return Register();
10124 if ((Subtarget->isNeonAvailable())) {
10125 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10126 }
10127 return Register();
10128}
10129
10130Register fastEmit_ISD_ADD_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10131 if (RetVT.SimpleTy != MVT::nxv16i8)
10132 return Register();
10133 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10134 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
10135 }
10136 return Register();
10137}
10138
10139Register fastEmit_ISD_ADD_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10140 if (RetVT.SimpleTy != MVT::nxv8i16)
10141 return Register();
10142 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10143 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10144 }
10145 return Register();
10146}
10147
10148Register fastEmit_ISD_ADD_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10149 if (RetVT.SimpleTy != MVT::nxv4i32)
10150 return Register();
10151 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10152 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10153 }
10154 return Register();
10155}
10156
10157Register fastEmit_ISD_ADD_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10158 if (RetVT.SimpleTy != MVT::nxv2i64)
10159 return Register();
10160 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10161 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10162 }
10163 return Register();
10164}
10165
10166Register fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10167 switch (VT.SimpleTy) {
10168 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
10169 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op1);
10170 case MVT::v8i8: return fastEmit_ISD_ADD_MVT_v8i8_rr(RetVT, Op0, Op1);
10171 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
10172 case MVT::v4i16: return fastEmit_ISD_ADD_MVT_v4i16_rr(RetVT, Op0, Op1);
10173 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
10174 case MVT::v2i32: return fastEmit_ISD_ADD_MVT_v2i32_rr(RetVT, Op0, Op1);
10175 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
10176 case MVT::v1i64: return fastEmit_ISD_ADD_MVT_v1i64_rr(RetVT, Op0, Op1);
10177 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
10178 case MVT::nxv16i8: return fastEmit_ISD_ADD_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10179 case MVT::nxv8i16: return fastEmit_ISD_ADD_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10180 case MVT::nxv4i32: return fastEmit_ISD_ADD_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10181 case MVT::nxv2i64: return fastEmit_ISD_ADD_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10182 default: return Register();
10183 }
10184}
10185
10186// FastEmit functions for ISD::AND.
10187
10188Register fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10189 if (RetVT.SimpleTy != MVT::i32)
10190 return Register();
10191 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
10192}
10193
10194Register fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10195 if (RetVT.SimpleTy != MVT::i64)
10196 return Register();
10197 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
10198}
10199
10200Register fastEmit_ISD_AND_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10201 if (RetVT.SimpleTy != MVT::v8i8)
10202 return Register();
10203 if ((Subtarget->isNeonAvailable())) {
10204 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10205 }
10206 return Register();
10207}
10208
10209Register fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10210 if (RetVT.SimpleTy != MVT::v16i8)
10211 return Register();
10212 if ((Subtarget->isNeonAvailable())) {
10213 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10214 }
10215 return Register();
10216}
10217
10218Register fastEmit_ISD_AND_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10219 if (RetVT.SimpleTy != MVT::v4i16)
10220 return Register();
10221 if ((Subtarget->isNeonAvailable())) {
10222 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10223 }
10224 return Register();
10225}
10226
10227Register fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10228 if (RetVT.SimpleTy != MVT::v8i16)
10229 return Register();
10230 if ((Subtarget->isNeonAvailable())) {
10231 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10232 }
10233 return Register();
10234}
10235
10236Register fastEmit_ISD_AND_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10237 if (RetVT.SimpleTy != MVT::v2i32)
10238 return Register();
10239 if ((Subtarget->isNeonAvailable())) {
10240 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10241 }
10242 return Register();
10243}
10244
10245Register fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10246 if (RetVT.SimpleTy != MVT::v4i32)
10247 return Register();
10248 if ((Subtarget->isNeonAvailable())) {
10249 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10250 }
10251 return Register();
10252}
10253
10254Register fastEmit_ISD_AND_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
10255 if (RetVT.SimpleTy != MVT::v1i64)
10256 return Register();
10257 if ((Subtarget->isNeonAvailable())) {
10258 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10259 }
10260 return Register();
10261}
10262
10263Register fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10264 if (RetVT.SimpleTy != MVT::v2i64)
10265 return Register();
10266 if ((Subtarget->isNeonAvailable())) {
10267 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10268 }
10269 return Register();
10270}
10271
10272Register fastEmit_ISD_AND_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10273 if (RetVT.SimpleTy != MVT::nxv16i8)
10274 return Register();
10275 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10276 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10277 }
10278 return Register();
10279}
10280
10281Register fastEmit_ISD_AND_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10282 if (RetVT.SimpleTy != MVT::nxv8i16)
10283 return Register();
10284 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10285 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10286 }
10287 return Register();
10288}
10289
10290Register fastEmit_ISD_AND_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10291 if (RetVT.SimpleTy != MVT::nxv4i32)
10292 return Register();
10293 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10294 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10295 }
10296 return Register();
10297}
10298
10299Register fastEmit_ISD_AND_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10300 if (RetVT.SimpleTy != MVT::nxv2i64)
10301 return Register();
10302 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10303 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10304 }
10305 return Register();
10306}
10307
10308Register fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10309 switch (VT.SimpleTy) {
10310 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
10311 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op1);
10312 case MVT::v8i8: return fastEmit_ISD_AND_MVT_v8i8_rr(RetVT, Op0, Op1);
10313 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
10314 case MVT::v4i16: return fastEmit_ISD_AND_MVT_v4i16_rr(RetVT, Op0, Op1);
10315 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
10316 case MVT::v2i32: return fastEmit_ISD_AND_MVT_v2i32_rr(RetVT, Op0, Op1);
10317 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
10318 case MVT::v1i64: return fastEmit_ISD_AND_MVT_v1i64_rr(RetVT, Op0, Op1);
10319 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
10320 case MVT::nxv16i8: return fastEmit_ISD_AND_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10321 case MVT::nxv8i16: return fastEmit_ISD_AND_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10322 case MVT::nxv4i32: return fastEmit_ISD_AND_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10323 case MVT::nxv2i64: return fastEmit_ISD_AND_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10324 default: return Register();
10325 }
10326}
10327
10328// FastEmit functions for ISD::AVGCEILS.
10329
10330Register fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10331 if (RetVT.SimpleTy != MVT::v8i8)
10332 return Register();
10333 if ((Subtarget->isNeonAvailable())) {
10334 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10335 }
10336 return Register();
10337}
10338
10339Register fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10340 if (RetVT.SimpleTy != MVT::v16i8)
10341 return Register();
10342 if ((Subtarget->isNeonAvailable())) {
10343 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10344 }
10345 return Register();
10346}
10347
10348Register fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10349 if (RetVT.SimpleTy != MVT::v4i16)
10350 return Register();
10351 if ((Subtarget->isNeonAvailable())) {
10352 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10353 }
10354 return Register();
10355}
10356
10357Register fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10358 if (RetVT.SimpleTy != MVT::v8i16)
10359 return Register();
10360 if ((Subtarget->isNeonAvailable())) {
10361 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10362 }
10363 return Register();
10364}
10365
10366Register fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10367 if (RetVT.SimpleTy != MVT::v2i32)
10368 return Register();
10369 if ((Subtarget->isNeonAvailable())) {
10370 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10371 }
10372 return Register();
10373}
10374
10375Register fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10376 if (RetVT.SimpleTy != MVT::v4i32)
10377 return Register();
10378 if ((Subtarget->isNeonAvailable())) {
10379 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10380 }
10381 return Register();
10382}
10383
10384Register fastEmit_ISD_AVGCEILS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10385 switch (VT.SimpleTy) {
10386 case MVT::v8i8: return fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(RetVT, Op0, Op1);
10387 case MVT::v16i8: return fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(RetVT, Op0, Op1);
10388 case MVT::v4i16: return fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(RetVT, Op0, Op1);
10389 case MVT::v8i16: return fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(RetVT, Op0, Op1);
10390 case MVT::v2i32: return fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(RetVT, Op0, Op1);
10391 case MVT::v4i32: return fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(RetVT, Op0, Op1);
10392 default: return Register();
10393 }
10394}
10395
10396// FastEmit functions for ISD::AVGCEILU.
10397
10398Register fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10399 if (RetVT.SimpleTy != MVT::v8i8)
10400 return Register();
10401 if ((Subtarget->isNeonAvailable())) {
10402 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10403 }
10404 return Register();
10405}
10406
10407Register fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10408 if (RetVT.SimpleTy != MVT::v16i8)
10409 return Register();
10410 if ((Subtarget->isNeonAvailable())) {
10411 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10412 }
10413 return Register();
10414}
10415
10416Register fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10417 if (RetVT.SimpleTy != MVT::v4i16)
10418 return Register();
10419 if ((Subtarget->isNeonAvailable())) {
10420 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10421 }
10422 return Register();
10423}
10424
10425Register fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10426 if (RetVT.SimpleTy != MVT::v8i16)
10427 return Register();
10428 if ((Subtarget->isNeonAvailable())) {
10429 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10430 }
10431 return Register();
10432}
10433
10434Register fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10435 if (RetVT.SimpleTy != MVT::v2i32)
10436 return Register();
10437 if ((Subtarget->isNeonAvailable())) {
10438 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10439 }
10440 return Register();
10441}
10442
10443Register fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10444 if (RetVT.SimpleTy != MVT::v4i32)
10445 return Register();
10446 if ((Subtarget->isNeonAvailable())) {
10447 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10448 }
10449 return Register();
10450}
10451
10452Register fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10453 switch (VT.SimpleTy) {
10454 case MVT::v8i8: return fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(RetVT, Op0, Op1);
10455 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
10456 case MVT::v4i16: return fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(RetVT, Op0, Op1);
10457 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
10458 case MVT::v2i32: return fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(RetVT, Op0, Op1);
10459 case MVT::v4i32: return fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(RetVT, Op0, Op1);
10460 default: return Register();
10461 }
10462}
10463
10464// FastEmit functions for ISD::AVGFLOORS.
10465
10466Register fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10467 if (RetVT.SimpleTy != MVT::v8i8)
10468 return Register();
10469 if ((Subtarget->isNeonAvailable())) {
10470 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10471 }
10472 return Register();
10473}
10474
10475Register fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10476 if (RetVT.SimpleTy != MVT::v16i8)
10477 return Register();
10478 if ((Subtarget->isNeonAvailable())) {
10479 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10480 }
10481 return Register();
10482}
10483
10484Register fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10485 if (RetVT.SimpleTy != MVT::v4i16)
10486 return Register();
10487 if ((Subtarget->isNeonAvailable())) {
10488 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10489 }
10490 return Register();
10491}
10492
10493Register fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10494 if (RetVT.SimpleTy != MVT::v8i16)
10495 return Register();
10496 if ((Subtarget->isNeonAvailable())) {
10497 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10498 }
10499 return Register();
10500}
10501
10502Register fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10503 if (RetVT.SimpleTy != MVT::v2i32)
10504 return Register();
10505 if ((Subtarget->isNeonAvailable())) {
10506 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10507 }
10508 return Register();
10509}
10510
10511Register fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10512 if (RetVT.SimpleTy != MVT::v4i32)
10513 return Register();
10514 if ((Subtarget->isNeonAvailable())) {
10515 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10516 }
10517 return Register();
10518}
10519
10520Register fastEmit_ISD_AVGFLOORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10521 switch (VT.SimpleTy) {
10522 case MVT::v8i8: return fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(RetVT, Op0, Op1);
10523 case MVT::v16i8: return fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(RetVT, Op0, Op1);
10524 case MVT::v4i16: return fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(RetVT, Op0, Op1);
10525 case MVT::v8i16: return fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(RetVT, Op0, Op1);
10526 case MVT::v2i32: return fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(RetVT, Op0, Op1);
10527 case MVT::v4i32: return fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(RetVT, Op0, Op1);
10528 default: return Register();
10529 }
10530}
10531
10532// FastEmit functions for ISD::AVGFLOORU.
10533
10534Register fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10535 if (RetVT.SimpleTy != MVT::v8i8)
10536 return Register();
10537 if ((Subtarget->isNeonAvailable())) {
10538 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10539 }
10540 return Register();
10541}
10542
10543Register fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10544 if (RetVT.SimpleTy != MVT::v16i8)
10545 return Register();
10546 if ((Subtarget->isNeonAvailable())) {
10547 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10548 }
10549 return Register();
10550}
10551
10552Register fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10553 if (RetVT.SimpleTy != MVT::v4i16)
10554 return Register();
10555 if ((Subtarget->isNeonAvailable())) {
10556 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10557 }
10558 return Register();
10559}
10560
10561Register fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10562 if (RetVT.SimpleTy != MVT::v8i16)
10563 return Register();
10564 if ((Subtarget->isNeonAvailable())) {
10565 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10566 }
10567 return Register();
10568}
10569
10570Register fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10571 if (RetVT.SimpleTy != MVT::v2i32)
10572 return Register();
10573 if ((Subtarget->isNeonAvailable())) {
10574 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10575 }
10576 return Register();
10577}
10578
10579Register fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10580 if (RetVT.SimpleTy != MVT::v4i32)
10581 return Register();
10582 if ((Subtarget->isNeonAvailable())) {
10583 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10584 }
10585 return Register();
10586}
10587
10588Register fastEmit_ISD_AVGFLOORU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10589 switch (VT.SimpleTy) {
10590 case MVT::v8i8: return fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(RetVT, Op0, Op1);
10591 case MVT::v16i8: return fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(RetVT, Op0, Op1);
10592 case MVT::v4i16: return fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(RetVT, Op0, Op1);
10593 case MVT::v8i16: return fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(RetVT, Op0, Op1);
10594 case MVT::v2i32: return fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(RetVT, Op0, Op1);
10595 case MVT::v4i32: return fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(RetVT, Op0, Op1);
10596 default: return Register();
10597 }
10598}
10599
10600// FastEmit functions for ISD::CLMUL.
10601
10602Register fastEmit_ISD_CLMUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10603 if (RetVT.SimpleTy != MVT::v8i8)
10604 return Register();
10605 if ((Subtarget->isNeonAvailable())) {
10606 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10607 }
10608 return Register();
10609}
10610
10611Register fastEmit_ISD_CLMUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10612 if (RetVT.SimpleTy != MVT::v16i8)
10613 return Register();
10614 if ((Subtarget->isNeonAvailable())) {
10615 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10616 }
10617 return Register();
10618}
10619
10620Register fastEmit_ISD_CLMUL_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10621 if (RetVT.SimpleTy != MVT::nxv16i8)
10622 return Register();
10623 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2() || Subtarget->hasSME()))) {
10624 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMUL_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
10625 }
10626 return Register();
10627}
10628
10629Register fastEmit_ISD_CLMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10630 switch (VT.SimpleTy) {
10631 case MVT::v8i8: return fastEmit_ISD_CLMUL_MVT_v8i8_rr(RetVT, Op0, Op1);
10632 case MVT::v16i8: return fastEmit_ISD_CLMUL_MVT_v16i8_rr(RetVT, Op0, Op1);
10633 case MVT::nxv16i8: return fastEmit_ISD_CLMUL_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10634 default: return Register();
10635 }
10636}
10637
10638// FastEmit functions for ISD::CONCAT_VECTORS.
10639
10640Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(MVT RetVT, Register Op0, Register Op1) {
10641 if (RetVT.SimpleTy != MVT::nxv2i1)
10642 return Register();
10643 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10644 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
10645 }
10646 return Register();
10647}
10648
10649Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
10650 if (RetVT.SimpleTy != MVT::nxv4i1)
10651 return Register();
10652 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10653 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
10654 }
10655 return Register();
10656}
10657
10658Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
10659 if (RetVT.SimpleTy != MVT::nxv8i1)
10660 return Register();
10661 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10662 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
10663 }
10664 return Register();
10665}
10666
10667Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
10668 if (RetVT.SimpleTy != MVT::nxv16i1)
10669 return Register();
10670 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10671 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
10672 }
10673 return Register();
10674}
10675
10676Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
10677 if (RetVT.SimpleTy != MVT::nxv4f16)
10678 return Register();
10679 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10680 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10681 }
10682 return Register();
10683}
10684
10685Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10686 if (RetVT.SimpleTy != MVT::nxv8f16)
10687 return Register();
10688 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10689 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10690 }
10691 return Register();
10692}
10693
10694Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10695 if (RetVT.SimpleTy != MVT::nxv4bf16)
10696 return Register();
10697 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10698 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10699 }
10700 return Register();
10701}
10702
10703Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10704 if (RetVT.SimpleTy != MVT::nxv8bf16)
10705 return Register();
10706 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10707 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10708 }
10709 return Register();
10710}
10711
10712Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10713 if (RetVT.SimpleTy != MVT::nxv4f32)
10714 return Register();
10715 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10716 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10717 }
10718 return Register();
10719}
10720
10721Register fastEmit_ISD_CONCAT_VECTORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10722 switch (VT.SimpleTy) {
10723 case MVT::nxv1i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(RetVT, Op0, Op1);
10724 case MVT::nxv2i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(RetVT, Op0, Op1);
10725 case MVT::nxv4i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(RetVT, Op0, Op1);
10726 case MVT::nxv8i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(RetVT, Op0, Op1);
10727 case MVT::nxv2f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(RetVT, Op0, Op1);
10728 case MVT::nxv4f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(RetVT, Op0, Op1);
10729 case MVT::nxv2bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
10730 case MVT::nxv4bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
10731 case MVT::nxv2f32: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(RetVT, Op0, Op1);
10732 default: return Register();
10733 }
10734}
10735
10736// FastEmit functions for ISD::FADD.
10737
10738Register fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10739 if (RetVT.SimpleTy != MVT::f16)
10740 return Register();
10741 if ((Subtarget->hasFullFP16())) {
10742 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
10743 }
10744 return Register();
10745}
10746
10747Register fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10748 if (RetVT.SimpleTy != MVT::f32)
10749 return Register();
10750 if ((Subtarget->hasFPARMv8())) {
10751 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
10752 }
10753 return Register();
10754}
10755
10756Register fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
10757 if (RetVT.SimpleTy != MVT::f64)
10758 return Register();
10759 if ((Subtarget->hasFPARMv8())) {
10760 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10761 }
10762 return Register();
10763}
10764
10765Register fastEmit_ISD_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10766 if (RetVT.SimpleTy != MVT::v4f16)
10767 return Register();
10768 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10769 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10770 }
10771 return Register();
10772}
10773
10774Register fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10775 if (RetVT.SimpleTy != MVT::v8f16)
10776 return Register();
10777 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10778 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10779 }
10780 return Register();
10781}
10782
10783Register fastEmit_ISD_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10784 if (RetVT.SimpleTy != MVT::v2f32)
10785 return Register();
10786 if ((Subtarget->isNeonAvailable())) {
10787 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10788 }
10789 return Register();
10790}
10791
10792Register fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10793 if (RetVT.SimpleTy != MVT::v4f32)
10794 return Register();
10795 if ((Subtarget->isNeonAvailable())) {
10796 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10797 }
10798 return Register();
10799}
10800
10801Register fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10802 if (RetVT.SimpleTy != MVT::v2f64)
10803 return Register();
10804 if ((Subtarget->isNeonAvailable())) {
10805 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10806 }
10807 return Register();
10808}
10809
10810Register fastEmit_ISD_FADD_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10811 if (RetVT.SimpleTy != MVT::nxv8f16)
10812 return Register();
10813 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10814 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10815 }
10816 return Register();
10817}
10818
10819Register fastEmit_ISD_FADD_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10820 if (RetVT.SimpleTy != MVT::nxv8bf16)
10821 return Register();
10822 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
10823 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFADD_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10824 }
10825 return Register();
10826}
10827
10828Register fastEmit_ISD_FADD_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10829 if (RetVT.SimpleTy != MVT::nxv4f32)
10830 return Register();
10831 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10832 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10833 }
10834 return Register();
10835}
10836
10837Register fastEmit_ISD_FADD_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10838 if (RetVT.SimpleTy != MVT::nxv2f64)
10839 return Register();
10840 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10841 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10842 }
10843 return Register();
10844}
10845
10846Register fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10847 switch (VT.SimpleTy) {
10848 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
10849 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
10850 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
10851 case MVT::v4f16: return fastEmit_ISD_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
10852 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
10853 case MVT::v2f32: return fastEmit_ISD_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
10854 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
10855 case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
10856 case MVT::nxv8f16: return fastEmit_ISD_FADD_MVT_nxv8f16_rr(RetVT, Op0, Op1);
10857 case MVT::nxv8bf16: return fastEmit_ISD_FADD_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
10858 case MVT::nxv4f32: return fastEmit_ISD_FADD_MVT_nxv4f32_rr(RetVT, Op0, Op1);
10859 case MVT::nxv2f64: return fastEmit_ISD_FADD_MVT_nxv2f64_rr(RetVT, Op0, Op1);
10860 default: return Register();
10861 }
10862}
10863
10864// FastEmit functions for ISD::FDIV.
10865
10866Register fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10867 if (RetVT.SimpleTy != MVT::f16)
10868 return Register();
10869 if ((Subtarget->hasFullFP16())) {
10870 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
10871 }
10872 return Register();
10873}
10874
10875Register fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10876 if (RetVT.SimpleTy != MVT::f32)
10877 return Register();
10878 if ((Subtarget->hasFPARMv8())) {
10879 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
10880 }
10881 return Register();
10882}
10883
10884Register fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
10885 if (RetVT.SimpleTy != MVT::f64)
10886 return Register();
10887 if ((Subtarget->hasFPARMv8())) {
10888 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10889 }
10890 return Register();
10891}
10892
10893Register fastEmit_ISD_FDIV_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10894 if (RetVT.SimpleTy != MVT::v4f16)
10895 return Register();
10896 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10897 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10898 }
10899 return Register();
10900}
10901
10902Register fastEmit_ISD_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10903 if (RetVT.SimpleTy != MVT::v8f16)
10904 return Register();
10905 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10906 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10907 }
10908 return Register();
10909}
10910
10911Register fastEmit_ISD_FDIV_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10912 if (RetVT.SimpleTy != MVT::v2f32)
10913 return Register();
10914 if ((Subtarget->isNeonAvailable())) {
10915 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10916 }
10917 return Register();
10918}
10919
10920Register fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10921 if (RetVT.SimpleTy != MVT::v4f32)
10922 return Register();
10923 if ((Subtarget->isNeonAvailable())) {
10924 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10925 }
10926 return Register();
10927}
10928
10929Register fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10930 if (RetVT.SimpleTy != MVT::v2f64)
10931 return Register();
10932 if ((Subtarget->isNeonAvailable())) {
10933 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10934 }
10935 return Register();
10936}
10937
10938Register fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10939 switch (VT.SimpleTy) {
10940 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
10941 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
10942 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
10943 case MVT::v4f16: return fastEmit_ISD_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1);
10944 case MVT::v8f16: return fastEmit_ISD_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
10945 case MVT::v2f32: return fastEmit_ISD_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1);
10946 case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
10947 case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
10948 default: return Register();
10949 }
10950}
10951
10952// FastEmit functions for ISD::FMAXIMUM.
10953
10954Register fastEmit_ISD_FMAXIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10955 if (RetVT.SimpleTy != MVT::f16)
10956 return Register();
10957 if ((Subtarget->hasFullFP16())) {
10958 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
10959 }
10960 return Register();
10961}
10962
10963Register fastEmit_ISD_FMAXIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10964 if (RetVT.SimpleTy != MVT::f32)
10965 return Register();
10966 if ((Subtarget->hasFPARMv8())) {
10967 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
10968 }
10969 return Register();
10970}
10971
10972Register fastEmit_ISD_FMAXIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
10973 if (RetVT.SimpleTy != MVT::f64)
10974 return Register();
10975 if ((Subtarget->hasFPARMv8())) {
10976 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10977 }
10978 return Register();
10979}
10980
10981Register fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10982 if (RetVT.SimpleTy != MVT::v4f16)
10983 return Register();
10984 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10985 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10986 }
10987 return Register();
10988}
10989
10990Register fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10991 if (RetVT.SimpleTy != MVT::v8f16)
10992 return Register();
10993 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10994 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10995 }
10996 return Register();
10997}
10998
10999Register fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11000 if (RetVT.SimpleTy != MVT::v2f32)
11001 return Register();
11002 if ((Subtarget->isNeonAvailable())) {
11003 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11004 }
11005 return Register();
11006}
11007
11008Register fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11009 if (RetVT.SimpleTy != MVT::v4f32)
11010 return Register();
11011 if ((Subtarget->isNeonAvailable())) {
11012 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11013 }
11014 return Register();
11015}
11016
11017Register fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11018 if (RetVT.SimpleTy != MVT::v1f64)
11019 return Register();
11020 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11021}
11022
11023Register fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11024 if (RetVT.SimpleTy != MVT::v2f64)
11025 return Register();
11026 if ((Subtarget->isNeonAvailable())) {
11027 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11028 }
11029 return Register();
11030}
11031
11032Register fastEmit_ISD_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11033 switch (VT.SimpleTy) {
11034 case MVT::f16: return fastEmit_ISD_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1);
11035 case MVT::f32: return fastEmit_ISD_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1);
11036 case MVT::f64: return fastEmit_ISD_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1);
11037 case MVT::v4f16: return fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11038 case MVT::v8f16: return fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11039 case MVT::v2f32: return fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11040 case MVT::v4f32: return fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11041 case MVT::v1f64: return fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11042 case MVT::v2f64: return fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11043 default: return Register();
11044 }
11045}
11046
11047// FastEmit functions for ISD::FMAXNUM.
11048
11049Register fastEmit_ISD_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11050 if (RetVT.SimpleTy != MVT::f16)
11051 return Register();
11052 if ((Subtarget->hasFullFP16())) {
11053 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11054 }
11055 return Register();
11056}
11057
11058Register fastEmit_ISD_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11059 if (RetVT.SimpleTy != MVT::f32)
11060 return Register();
11061 if ((Subtarget->hasFPARMv8())) {
11062 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11063 }
11064 return Register();
11065}
11066
11067Register fastEmit_ISD_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11068 if (RetVT.SimpleTy != MVT::f64)
11069 return Register();
11070 if ((Subtarget->hasFPARMv8())) {
11071 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11072 }
11073 return Register();
11074}
11075
11076Register fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11077 if (RetVT.SimpleTy != MVT::v4f16)
11078 return Register();
11079 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11080 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11081 }
11082 return Register();
11083}
11084
11085Register fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11086 if (RetVT.SimpleTy != MVT::v8f16)
11087 return Register();
11088 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11089 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11090 }
11091 return Register();
11092}
11093
11094Register fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11095 if (RetVT.SimpleTy != MVT::v2f32)
11096 return Register();
11097 if ((Subtarget->isNeonAvailable())) {
11098 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11099 }
11100 return Register();
11101}
11102
11103Register fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11104 if (RetVT.SimpleTy != MVT::v4f32)
11105 return Register();
11106 if ((Subtarget->isNeonAvailable())) {
11107 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11108 }
11109 return Register();
11110}
11111
11112Register fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11113 if (RetVT.SimpleTy != MVT::v1f64)
11114 return Register();
11115 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11116}
11117
11118Register fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11119 if (RetVT.SimpleTy != MVT::v2f64)
11120 return Register();
11121 if ((Subtarget->isNeonAvailable())) {
11122 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11123 }
11124 return Register();
11125}
11126
11127Register fastEmit_ISD_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11128 switch (VT.SimpleTy) {
11129 case MVT::f16: return fastEmit_ISD_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
11130 case MVT::f32: return fastEmit_ISD_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
11131 case MVT::f64: return fastEmit_ISD_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
11132 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11133 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11134 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11135 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11136 case MVT::v1f64: return fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11137 case MVT::v2f64: return fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11138 default: return Register();
11139 }
11140}
11141
11142// FastEmit functions for ISD::FMAXNUM_IEEE.
11143
11144Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11145 if (RetVT.SimpleTy != MVT::f16)
11146 return Register();
11147 if ((Subtarget->hasFullFP16())) {
11148 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11149 }
11150 return Register();
11151}
11152
11153Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11154 if (RetVT.SimpleTy != MVT::f32)
11155 return Register();
11156 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11157}
11158
11159Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11160 if (RetVT.SimpleTy != MVT::f64)
11161 return Register();
11162 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11163}
11164
11165Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11166 if (RetVT.SimpleTy != MVT::v4f16)
11167 return Register();
11168 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11169 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11170 }
11171 return Register();
11172}
11173
11174Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11175 if (RetVT.SimpleTy != MVT::v8f16)
11176 return Register();
11177 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11178 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11179 }
11180 return Register();
11181}
11182
11183Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11184 if (RetVT.SimpleTy != MVT::v2f32)
11185 return Register();
11186 if ((Subtarget->isNeonAvailable())) {
11187 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11188 }
11189 return Register();
11190}
11191
11192Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11193 if (RetVT.SimpleTy != MVT::v4f32)
11194 return Register();
11195 if ((Subtarget->isNeonAvailable())) {
11196 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11197 }
11198 return Register();
11199}
11200
11201Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11202 if (RetVT.SimpleTy != MVT::v2f64)
11203 return Register();
11204 if ((Subtarget->isNeonAvailable())) {
11205 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11206 }
11207 return Register();
11208}
11209
11210Register fastEmit_ISD_FMAXNUM_IEEE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11211 switch (VT.SimpleTy) {
11212 case MVT::f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f16_rr(RetVT, Op0, Op1);
11213 case MVT::f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f32_rr(RetVT, Op0, Op1);
11214 case MVT::f64: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f64_rr(RetVT, Op0, Op1);
11215 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f16_rr(RetVT, Op0, Op1);
11216 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v8f16_rr(RetVT, Op0, Op1);
11217 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f32_rr(RetVT, Op0, Op1);
11218 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f32_rr(RetVT, Op0, Op1);
11219 case MVT::v2f64: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f64_rr(RetVT, Op0, Op1);
11220 default: return Register();
11221 }
11222}
11223
11224// FastEmit functions for ISD::FMINIMUM.
11225
11226Register fastEmit_ISD_FMINIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11227 if (RetVT.SimpleTy != MVT::f16)
11228 return Register();
11229 if ((Subtarget->hasFullFP16())) {
11230 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11231 }
11232 return Register();
11233}
11234
11235Register fastEmit_ISD_FMINIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11236 if (RetVT.SimpleTy != MVT::f32)
11237 return Register();
11238 if ((Subtarget->hasFPARMv8())) {
11239 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11240 }
11241 return Register();
11242}
11243
11244Register fastEmit_ISD_FMINIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11245 if (RetVT.SimpleTy != MVT::f64)
11246 return Register();
11247 if ((Subtarget->hasFPARMv8())) {
11248 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11249 }
11250 return Register();
11251}
11252
11253Register fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11254 if (RetVT.SimpleTy != MVT::v4f16)
11255 return Register();
11256 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11257 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11258 }
11259 return Register();
11260}
11261
11262Register fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11263 if (RetVT.SimpleTy != MVT::v8f16)
11264 return Register();
11265 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11266 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11267 }
11268 return Register();
11269}
11270
11271Register fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11272 if (RetVT.SimpleTy != MVT::v2f32)
11273 return Register();
11274 if ((Subtarget->isNeonAvailable())) {
11275 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11276 }
11277 return Register();
11278}
11279
11280Register fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11281 if (RetVT.SimpleTy != MVT::v4f32)
11282 return Register();
11283 if ((Subtarget->isNeonAvailable())) {
11284 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11285 }
11286 return Register();
11287}
11288
11289Register fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11290 if (RetVT.SimpleTy != MVT::v1f64)
11291 return Register();
11292 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11293}
11294
11295Register fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11296 if (RetVT.SimpleTy != MVT::v2f64)
11297 return Register();
11298 if ((Subtarget->isNeonAvailable())) {
11299 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11300 }
11301 return Register();
11302}
11303
11304Register fastEmit_ISD_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11305 switch (VT.SimpleTy) {
11306 case MVT::f16: return fastEmit_ISD_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1);
11307 case MVT::f32: return fastEmit_ISD_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1);
11308 case MVT::f64: return fastEmit_ISD_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1);
11309 case MVT::v4f16: return fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11310 case MVT::v8f16: return fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11311 case MVT::v2f32: return fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11312 case MVT::v4f32: return fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11313 case MVT::v1f64: return fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11314 case MVT::v2f64: return fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11315 default: return Register();
11316 }
11317}
11318
11319// FastEmit functions for ISD::FMINNUM.
11320
11321Register fastEmit_ISD_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11322 if (RetVT.SimpleTy != MVT::f16)
11323 return Register();
11324 if ((Subtarget->hasFullFP16())) {
11325 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11326 }
11327 return Register();
11328}
11329
11330Register fastEmit_ISD_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11331 if (RetVT.SimpleTy != MVT::f32)
11332 return Register();
11333 if ((Subtarget->hasFPARMv8())) {
11334 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11335 }
11336 return Register();
11337}
11338
11339Register fastEmit_ISD_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11340 if (RetVT.SimpleTy != MVT::f64)
11341 return Register();
11342 if ((Subtarget->hasFPARMv8())) {
11343 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11344 }
11345 return Register();
11346}
11347
11348Register fastEmit_ISD_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11349 if (RetVT.SimpleTy != MVT::v4f16)
11350 return Register();
11351 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11352 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11353 }
11354 return Register();
11355}
11356
11357Register fastEmit_ISD_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11358 if (RetVT.SimpleTy != MVT::v8f16)
11359 return Register();
11360 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11361 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11362 }
11363 return Register();
11364}
11365
11366Register fastEmit_ISD_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11367 if (RetVT.SimpleTy != MVT::v2f32)
11368 return Register();
11369 if ((Subtarget->isNeonAvailable())) {
11370 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11371 }
11372 return Register();
11373}
11374
11375Register fastEmit_ISD_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11376 if (RetVT.SimpleTy != MVT::v4f32)
11377 return Register();
11378 if ((Subtarget->isNeonAvailable())) {
11379 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11380 }
11381 return Register();
11382}
11383
11384Register fastEmit_ISD_FMINNUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11385 if (RetVT.SimpleTy != MVT::v1f64)
11386 return Register();
11387 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11388}
11389
11390Register fastEmit_ISD_FMINNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11391 if (RetVT.SimpleTy != MVT::v2f64)
11392 return Register();
11393 if ((Subtarget->isNeonAvailable())) {
11394 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11395 }
11396 return Register();
11397}
11398
11399Register fastEmit_ISD_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11400 switch (VT.SimpleTy) {
11401 case MVT::f16: return fastEmit_ISD_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
11402 case MVT::f32: return fastEmit_ISD_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
11403 case MVT::f64: return fastEmit_ISD_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
11404 case MVT::v4f16: return fastEmit_ISD_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11405 case MVT::v8f16: return fastEmit_ISD_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11406 case MVT::v2f32: return fastEmit_ISD_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11407 case MVT::v4f32: return fastEmit_ISD_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11408 case MVT::v1f64: return fastEmit_ISD_FMINNUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11409 case MVT::v2f64: return fastEmit_ISD_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11410 default: return Register();
11411 }
11412}
11413
11414// FastEmit functions for ISD::FMINNUM_IEEE.
11415
11416Register fastEmit_ISD_FMINNUM_IEEE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11417 if (RetVT.SimpleTy != MVT::f16)
11418 return Register();
11419 if ((Subtarget->hasFullFP16())) {
11420 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11421 }
11422 return Register();
11423}
11424
11425Register fastEmit_ISD_FMINNUM_IEEE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11426 if (RetVT.SimpleTy != MVT::f32)
11427 return Register();
11428 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11429}
11430
11431Register fastEmit_ISD_FMINNUM_IEEE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11432 if (RetVT.SimpleTy != MVT::f64)
11433 return Register();
11434 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11435}
11436
11437Register fastEmit_ISD_FMINNUM_IEEE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11438 if (RetVT.SimpleTy != MVT::v4f16)
11439 return Register();
11440 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11441 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11442 }
11443 return Register();
11444}
11445
11446Register fastEmit_ISD_FMINNUM_IEEE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11447 if (RetVT.SimpleTy != MVT::v8f16)
11448 return Register();
11449 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11450 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11451 }
11452 return Register();
11453}
11454
11455Register fastEmit_ISD_FMINNUM_IEEE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11456 if (RetVT.SimpleTy != MVT::v2f32)
11457 return Register();
11458 if ((Subtarget->isNeonAvailable())) {
11459 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11460 }
11461 return Register();
11462}
11463
11464Register fastEmit_ISD_FMINNUM_IEEE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11465 if (RetVT.SimpleTy != MVT::v4f32)
11466 return Register();
11467 if ((Subtarget->isNeonAvailable())) {
11468 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11469 }
11470 return Register();
11471}
11472
11473Register fastEmit_ISD_FMINNUM_IEEE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11474 if (RetVT.SimpleTy != MVT::v2f64)
11475 return Register();
11476 if ((Subtarget->isNeonAvailable())) {
11477 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11478 }
11479 return Register();
11480}
11481
11482Register fastEmit_ISD_FMINNUM_IEEE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11483 switch (VT.SimpleTy) {
11484 case MVT::f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_f16_rr(RetVT, Op0, Op1);
11485 case MVT::f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_f32_rr(RetVT, Op0, Op1);
11486 case MVT::f64: return fastEmit_ISD_FMINNUM_IEEE_MVT_f64_rr(RetVT, Op0, Op1);
11487 case MVT::v4f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_v4f16_rr(RetVT, Op0, Op1);
11488 case MVT::v8f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_v8f16_rr(RetVT, Op0, Op1);
11489 case MVT::v2f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_v2f32_rr(RetVT, Op0, Op1);
11490 case MVT::v4f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_v4f32_rr(RetVT, Op0, Op1);
11491 case MVT::v2f64: return fastEmit_ISD_FMINNUM_IEEE_MVT_v2f64_rr(RetVT, Op0, Op1);
11492 default: return Register();
11493 }
11494}
11495
11496// FastEmit functions for ISD::FMUL.
11497
11498Register fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11499 if (RetVT.SimpleTy != MVT::f16)
11500 return Register();
11501 if ((Subtarget->hasFullFP16())) {
11502 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11503 }
11504 return Register();
11505}
11506
11507Register fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11508 if (RetVT.SimpleTy != MVT::f32)
11509 return Register();
11510 if ((Subtarget->hasFPARMv8())) {
11511 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11512 }
11513 return Register();
11514}
11515
11516Register fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11517 if (RetVT.SimpleTy != MVT::f64)
11518 return Register();
11519 if ((Subtarget->hasFPARMv8())) {
11520 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11521 }
11522 return Register();
11523}
11524
11525Register fastEmit_ISD_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11526 if (RetVT.SimpleTy != MVT::v4f16)
11527 return Register();
11528 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11529 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11530 }
11531 return Register();
11532}
11533
11534Register fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11535 if (RetVT.SimpleTy != MVT::v8f16)
11536 return Register();
11537 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11538 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11539 }
11540 return Register();
11541}
11542
11543Register fastEmit_ISD_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11544 if (RetVT.SimpleTy != MVT::v2f32)
11545 return Register();
11546 if ((Subtarget->isNeonAvailable())) {
11547 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11548 }
11549 return Register();
11550}
11551
11552Register fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11553 if (RetVT.SimpleTy != MVT::v4f32)
11554 return Register();
11555 if ((Subtarget->isNeonAvailable())) {
11556 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11557 }
11558 return Register();
11559}
11560
11561Register fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11562 if (RetVT.SimpleTy != MVT::v2f64)
11563 return Register();
11564 if ((Subtarget->isNeonAvailable())) {
11565 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11566 }
11567 return Register();
11568}
11569
11570Register fastEmit_ISD_FMUL_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11571 if (RetVT.SimpleTy != MVT::nxv8f16)
11572 return Register();
11573 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11574 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11575 }
11576 return Register();
11577}
11578
11579Register fastEmit_ISD_FMUL_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11580 if (RetVT.SimpleTy != MVT::nxv8bf16)
11581 return Register();
11582 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
11583 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFMUL_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11584 }
11585 return Register();
11586}
11587
11588Register fastEmit_ISD_FMUL_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11589 if (RetVT.SimpleTy != MVT::nxv4f32)
11590 return Register();
11591 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11592 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11593 }
11594 return Register();
11595}
11596
11597Register fastEmit_ISD_FMUL_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11598 if (RetVT.SimpleTy != MVT::nxv2f64)
11599 return Register();
11600 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11601 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11602 }
11603 return Register();
11604}
11605
11606Register fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11607 switch (VT.SimpleTy) {
11608 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
11609 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
11610 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
11611 case MVT::v4f16: return fastEmit_ISD_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
11612 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
11613 case MVT::v2f32: return fastEmit_ISD_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
11614 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
11615 case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
11616 case MVT::nxv8f16: return fastEmit_ISD_FMUL_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11617 case MVT::nxv8bf16: return fastEmit_ISD_FMUL_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11618 case MVT::nxv4f32: return fastEmit_ISD_FMUL_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11619 case MVT::nxv2f64: return fastEmit_ISD_FMUL_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11620 default: return Register();
11621 }
11622}
11623
11624// FastEmit functions for ISD::FSUB.
11625
11626Register fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11627 if (RetVT.SimpleTy != MVT::f16)
11628 return Register();
11629 if ((Subtarget->hasFullFP16())) {
11630 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11631 }
11632 return Register();
11633}
11634
11635Register fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11636 if (RetVT.SimpleTy != MVT::f32)
11637 return Register();
11638 if ((Subtarget->hasFPARMv8())) {
11639 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11640 }
11641 return Register();
11642}
11643
11644Register fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11645 if (RetVT.SimpleTy != MVT::f64)
11646 return Register();
11647 if ((Subtarget->hasFPARMv8())) {
11648 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11649 }
11650 return Register();
11651}
11652
11653Register fastEmit_ISD_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11654 if (RetVT.SimpleTy != MVT::v4f16)
11655 return Register();
11656 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11657 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11658 }
11659 return Register();
11660}
11661
11662Register fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11663 if (RetVT.SimpleTy != MVT::v8f16)
11664 return Register();
11665 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11666 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11667 }
11668 return Register();
11669}
11670
11671Register fastEmit_ISD_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11672 if (RetVT.SimpleTy != MVT::v2f32)
11673 return Register();
11674 if ((Subtarget->isNeonAvailable())) {
11675 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11676 }
11677 return Register();
11678}
11679
11680Register fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11681 if (RetVT.SimpleTy != MVT::v4f32)
11682 return Register();
11683 if ((Subtarget->isNeonAvailable())) {
11684 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11685 }
11686 return Register();
11687}
11688
11689Register fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11690 if (RetVT.SimpleTy != MVT::v2f64)
11691 return Register();
11692 if ((Subtarget->isNeonAvailable())) {
11693 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11694 }
11695 return Register();
11696}
11697
11698Register fastEmit_ISD_FSUB_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11699 if (RetVT.SimpleTy != MVT::nxv8f16)
11700 return Register();
11701 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11702 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11703 }
11704 return Register();
11705}
11706
11707Register fastEmit_ISD_FSUB_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11708 if (RetVT.SimpleTy != MVT::nxv8bf16)
11709 return Register();
11710 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
11711 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFSUB_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11712 }
11713 return Register();
11714}
11715
11716Register fastEmit_ISD_FSUB_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11717 if (RetVT.SimpleTy != MVT::nxv4f32)
11718 return Register();
11719 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11720 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11721 }
11722 return Register();
11723}
11724
11725Register fastEmit_ISD_FSUB_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11726 if (RetVT.SimpleTy != MVT::nxv2f64)
11727 return Register();
11728 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11729 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11730 }
11731 return Register();
11732}
11733
11734Register fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11735 switch (VT.SimpleTy) {
11736 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
11737 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
11738 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
11739 case MVT::v4f16: return fastEmit_ISD_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
11740 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
11741 case MVT::v2f32: return fastEmit_ISD_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
11742 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
11743 case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
11744 case MVT::nxv8f16: return fastEmit_ISD_FSUB_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11745 case MVT::nxv8bf16: return fastEmit_ISD_FSUB_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11746 case MVT::nxv4f32: return fastEmit_ISD_FSUB_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11747 case MVT::nxv2f64: return fastEmit_ISD_FSUB_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11748 default: return Register();
11749 }
11750}
11751
11752// FastEmit functions for ISD::GET_ACTIVE_LANE_MASK.
11753
11754Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv2i1_rr(Register Op0, Register Op1) {
11755 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11756 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_D, RC: &AArch64::PPRRegClass, Op0, Op1);
11757 }
11758 return Register();
11759}
11760
11761Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv4i1_rr(Register Op0, Register Op1) {
11762 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11763 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_S, RC: &AArch64::PPRRegClass, Op0, Op1);
11764 }
11765 return Register();
11766}
11767
11768Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv8i1_rr(Register Op0, Register Op1) {
11769 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11770 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_H, RC: &AArch64::PPRRegClass, Op0, Op1);
11771 }
11772 return Register();
11773}
11774
11775Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv16i1_rr(Register Op0, Register Op1) {
11776 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11777 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_B, RC: &AArch64::PPRRegClass, Op0, Op1);
11778 }
11779 return Register();
11780}
11781
11782Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
11783switch (RetVT.SimpleTy) {
11784 case MVT::nxv2i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv2i1_rr(Op0, Op1);
11785 case MVT::nxv4i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv4i1_rr(Op0, Op1);
11786 case MVT::nxv8i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv8i1_rr(Op0, Op1);
11787 case MVT::nxv16i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv16i1_rr(Op0, Op1);
11788 default: return Register();
11789}
11790}
11791
11792Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv2i1_rr(Register Op0, Register Op1) {
11793 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11794 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_D, RC: &AArch64::PPRRegClass, Op0, Op1);
11795 }
11796 return Register();
11797}
11798
11799Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv4i1_rr(Register Op0, Register Op1) {
11800 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11801 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_S, RC: &AArch64::PPRRegClass, Op0, Op1);
11802 }
11803 return Register();
11804}
11805
11806Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv8i1_rr(Register Op0, Register Op1) {
11807 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11808 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_H, RC: &AArch64::PPRRegClass, Op0, Op1);
11809 }
11810 return Register();
11811}
11812
11813Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv16i1_rr(Register Op0, Register Op1) {
11814 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11815 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_B, RC: &AArch64::PPRRegClass, Op0, Op1);
11816 }
11817 return Register();
11818}
11819
11820Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11821switch (RetVT.SimpleTy) {
11822 case MVT::nxv2i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv2i1_rr(Op0, Op1);
11823 case MVT::nxv4i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv4i1_rr(Op0, Op1);
11824 case MVT::nxv8i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv8i1_rr(Op0, Op1);
11825 case MVT::nxv16i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv16i1_rr(Op0, Op1);
11826 default: return Register();
11827}
11828}
11829
11830Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11831 switch (VT.SimpleTy) {
11832 case MVT::i32: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_rr(RetVT, Op0, Op1);
11833 case MVT::i64: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_rr(RetVT, Op0, Op1);
11834 default: return Register();
11835 }
11836}
11837
11838// FastEmit functions for ISD::MUL.
11839
11840Register fastEmit_ISD_MUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
11841 if (RetVT.SimpleTy != MVT::v8i8)
11842 return Register();
11843 if ((Subtarget->isNeonAvailable())) {
11844 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
11845 }
11846 return Register();
11847}
11848
11849Register fastEmit_ISD_MUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
11850 if (RetVT.SimpleTy != MVT::v16i8)
11851 return Register();
11852 if ((Subtarget->isNeonAvailable())) {
11853 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
11854 }
11855 return Register();
11856}
11857
11858Register fastEmit_ISD_MUL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
11859 if (RetVT.SimpleTy != MVT::v4i16)
11860 return Register();
11861 if ((Subtarget->isNeonAvailable())) {
11862 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11863 }
11864 return Register();
11865}
11866
11867Register fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
11868 if (RetVT.SimpleTy != MVT::v8i16)
11869 return Register();
11870 if ((Subtarget->isNeonAvailable())) {
11871 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11872 }
11873 return Register();
11874}
11875
11876Register fastEmit_ISD_MUL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
11877 if (RetVT.SimpleTy != MVT::v2i32)
11878 return Register();
11879 if ((Subtarget->isNeonAvailable())) {
11880 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11881 }
11882 return Register();
11883}
11884
11885Register fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
11886 if (RetVT.SimpleTy != MVT::v4i32)
11887 return Register();
11888 if ((Subtarget->isNeonAvailable())) {
11889 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11890 }
11891 return Register();
11892}
11893
11894Register fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11895 switch (VT.SimpleTy) {
11896 case MVT::v8i8: return fastEmit_ISD_MUL_MVT_v8i8_rr(RetVT, Op0, Op1);
11897 case MVT::v16i8: return fastEmit_ISD_MUL_MVT_v16i8_rr(RetVT, Op0, Op1);
11898 case MVT::v4i16: return fastEmit_ISD_MUL_MVT_v4i16_rr(RetVT, Op0, Op1);
11899 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
11900 case MVT::v2i32: return fastEmit_ISD_MUL_MVT_v2i32_rr(RetVT, Op0, Op1);
11901 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
11902 default: return Register();
11903 }
11904}
11905
11906// FastEmit functions for ISD::MULHS.
11907
11908Register fastEmit_ISD_MULHS_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11909 if (RetVT.SimpleTy != MVT::i64)
11910 return Register();
11911 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULHrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
11912}
11913
11914Register fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11915 switch (VT.SimpleTy) {
11916 case MVT::i64: return fastEmit_ISD_MULHS_MVT_i64_rr(RetVT, Op0, Op1);
11917 default: return Register();
11918 }
11919}
11920
11921// FastEmit functions for ISD::MULHU.
11922
11923Register fastEmit_ISD_MULHU_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11924 if (RetVT.SimpleTy != MVT::i64)
11925 return Register();
11926 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULHrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
11927}
11928
11929Register fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11930 switch (VT.SimpleTy) {
11931 case MVT::i64: return fastEmit_ISD_MULHU_MVT_i64_rr(RetVT, Op0, Op1);
11932 default: return Register();
11933 }
11934}
11935
11936// FastEmit functions for ISD::OR.
11937
11938Register fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
11939 if (RetVT.SimpleTy != MVT::i32)
11940 return Register();
11941 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
11942}
11943
11944Register fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11945 if (RetVT.SimpleTy != MVT::i64)
11946 return Register();
11947 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
11948}
11949
11950Register fastEmit_ISD_OR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
11951 if (RetVT.SimpleTy != MVT::v8i8)
11952 return Register();
11953 if ((Subtarget->isNeonAvailable())) {
11954 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
11955 }
11956 return Register();
11957}
11958
11959Register fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
11960 if (RetVT.SimpleTy != MVT::v16i8)
11961 return Register();
11962 if ((Subtarget->isNeonAvailable())) {
11963 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
11964 }
11965 return Register();
11966}
11967
11968Register fastEmit_ISD_OR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
11969 if (RetVT.SimpleTy != MVT::v4i16)
11970 return Register();
11971 if ((Subtarget->isNeonAvailable())) {
11972 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
11973 }
11974 return Register();
11975}
11976
11977Register fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
11978 if (RetVT.SimpleTy != MVT::v8i16)
11979 return Register();
11980 if ((Subtarget->isNeonAvailable())) {
11981 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
11982 }
11983 return Register();
11984}
11985
11986Register fastEmit_ISD_OR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
11987 if (RetVT.SimpleTy != MVT::v2i32)
11988 return Register();
11989 if ((Subtarget->isNeonAvailable())) {
11990 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
11991 }
11992 return Register();
11993}
11994
11995Register fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
11996 if (RetVT.SimpleTy != MVT::v4i32)
11997 return Register();
11998 if ((Subtarget->isNeonAvailable())) {
11999 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12000 }
12001 return Register();
12002}
12003
12004Register fastEmit_ISD_OR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12005 if (RetVT.SimpleTy != MVT::v1i64)
12006 return Register();
12007 if ((Subtarget->isNeonAvailable())) {
12008 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12009 }
12010 return Register();
12011}
12012
12013Register fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12014 if (RetVT.SimpleTy != MVT::v2i64)
12015 return Register();
12016 if ((Subtarget->isNeonAvailable())) {
12017 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12018 }
12019 return Register();
12020}
12021
12022Register fastEmit_ISD_OR_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12023 if (RetVT.SimpleTy != MVT::nxv16i8)
12024 return Register();
12025 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12026 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12027 }
12028 return Register();
12029}
12030
12031Register fastEmit_ISD_OR_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12032 if (RetVT.SimpleTy != MVT::nxv8i16)
12033 return Register();
12034 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12035 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12036 }
12037 return Register();
12038}
12039
12040Register fastEmit_ISD_OR_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12041 if (RetVT.SimpleTy != MVT::nxv4i32)
12042 return Register();
12043 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12044 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12045 }
12046 return Register();
12047}
12048
12049Register fastEmit_ISD_OR_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12050 if (RetVT.SimpleTy != MVT::nxv2i64)
12051 return Register();
12052 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12053 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12054 }
12055 return Register();
12056}
12057
12058Register fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12059 switch (VT.SimpleTy) {
12060 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
12061 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op1);
12062 case MVT::v8i8: return fastEmit_ISD_OR_MVT_v8i8_rr(RetVT, Op0, Op1);
12063 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
12064 case MVT::v4i16: return fastEmit_ISD_OR_MVT_v4i16_rr(RetVT, Op0, Op1);
12065 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
12066 case MVT::v2i32: return fastEmit_ISD_OR_MVT_v2i32_rr(RetVT, Op0, Op1);
12067 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
12068 case MVT::v1i64: return fastEmit_ISD_OR_MVT_v1i64_rr(RetVT, Op0, Op1);
12069 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
12070 case MVT::nxv16i8: return fastEmit_ISD_OR_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12071 case MVT::nxv8i16: return fastEmit_ISD_OR_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12072 case MVT::nxv4i32: return fastEmit_ISD_OR_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12073 case MVT::nxv2i64: return fastEmit_ISD_OR_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12074 default: return Register();
12075 }
12076}
12077
12078// FastEmit functions for ISD::ROTR.
12079
12080Register fastEmit_ISD_ROTR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12081 if (RetVT.SimpleTy != MVT::i64)
12082 return Register();
12083 return fastEmitInst_rr(MachineInstOpcode: AArch64::RORVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12084}
12085
12086Register fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12087 switch (VT.SimpleTy) {
12088 case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_rr(RetVT, Op0, Op1);
12089 default: return Register();
12090 }
12091}
12092
12093// FastEmit functions for ISD::SADDSAT.
12094
12095Register fastEmit_ISD_SADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12096 if (RetVT.SimpleTy != MVT::v8i8)
12097 return Register();
12098 if ((Subtarget->isNeonAvailable())) {
12099 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12100 }
12101 return Register();
12102}
12103
12104Register fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12105 if (RetVT.SimpleTy != MVT::v16i8)
12106 return Register();
12107 if ((Subtarget->isNeonAvailable())) {
12108 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12109 }
12110 return Register();
12111}
12112
12113Register fastEmit_ISD_SADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12114 if (RetVT.SimpleTy != MVT::v4i16)
12115 return Register();
12116 if ((Subtarget->isNeonAvailable())) {
12117 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12118 }
12119 return Register();
12120}
12121
12122Register fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12123 if (RetVT.SimpleTy != MVT::v8i16)
12124 return Register();
12125 if ((Subtarget->isNeonAvailable())) {
12126 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12127 }
12128 return Register();
12129}
12130
12131Register fastEmit_ISD_SADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12132 if (RetVT.SimpleTy != MVT::v2i32)
12133 return Register();
12134 if ((Subtarget->isNeonAvailable())) {
12135 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12136 }
12137 return Register();
12138}
12139
12140Register fastEmit_ISD_SADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12141 if (RetVT.SimpleTy != MVT::v4i32)
12142 return Register();
12143 if ((Subtarget->isNeonAvailable())) {
12144 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12145 }
12146 return Register();
12147}
12148
12149Register fastEmit_ISD_SADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12150 if (RetVT.SimpleTy != MVT::v1i64)
12151 return Register();
12152 if ((Subtarget->isNeonAvailable())) {
12153 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
12154 }
12155 return Register();
12156}
12157
12158Register fastEmit_ISD_SADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12159 if (RetVT.SimpleTy != MVT::v2i64)
12160 return Register();
12161 if ((Subtarget->isNeonAvailable())) {
12162 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12163 }
12164 return Register();
12165}
12166
12167Register fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12168 if (RetVT.SimpleTy != MVT::nxv16i8)
12169 return Register();
12170 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12171 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
12172 }
12173 return Register();
12174}
12175
12176Register fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12177 if (RetVT.SimpleTy != MVT::nxv8i16)
12178 return Register();
12179 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12180 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
12181 }
12182 return Register();
12183}
12184
12185Register fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12186 if (RetVT.SimpleTy != MVT::nxv4i32)
12187 return Register();
12188 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12189 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
12190 }
12191 return Register();
12192}
12193
12194Register fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12195 if (RetVT.SimpleTy != MVT::nxv2i64)
12196 return Register();
12197 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12198 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
12199 }
12200 return Register();
12201}
12202
12203Register fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12204 switch (VT.SimpleTy) {
12205 case MVT::v8i8: return fastEmit_ISD_SADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
12206 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12207 case MVT::v4i16: return fastEmit_ISD_SADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
12208 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12209 case MVT::v2i32: return fastEmit_ISD_SADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
12210 case MVT::v4i32: return fastEmit_ISD_SADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
12211 case MVT::v1i64: return fastEmit_ISD_SADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
12212 case MVT::v2i64: return fastEmit_ISD_SADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
12213 case MVT::nxv16i8: return fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12214 case MVT::nxv8i16: return fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12215 case MVT::nxv4i32: return fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12216 case MVT::nxv2i64: return fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12217 default: return Register();
12218 }
12219}
12220
12221// FastEmit functions for ISD::SDIV.
12222
12223Register fastEmit_ISD_SDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12224 if (RetVT.SimpleTy != MVT::i32)
12225 return Register();
12226 return fastEmitInst_rr(MachineInstOpcode: AArch64::SDIVWr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12227}
12228
12229Register fastEmit_ISD_SDIV_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12230 if (RetVT.SimpleTy != MVT::i64)
12231 return Register();
12232 return fastEmitInst_rr(MachineInstOpcode: AArch64::SDIVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12233}
12234
12235Register fastEmit_ISD_SDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12236 switch (VT.SimpleTy) {
12237 case MVT::i32: return fastEmit_ISD_SDIV_MVT_i32_rr(RetVT, Op0, Op1);
12238 case MVT::i64: return fastEmit_ISD_SDIV_MVT_i64_rr(RetVT, Op0, Op1);
12239 default: return Register();
12240 }
12241}
12242
12243// FastEmit functions for ISD::SHL.
12244
12245Register fastEmit_ISD_SHL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12246 if (RetVT.SimpleTy != MVT::i64)
12247 return Register();
12248 return fastEmitInst_rr(MachineInstOpcode: AArch64::LSLVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12249}
12250
12251Register fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12252 switch (VT.SimpleTy) {
12253 case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_rr(RetVT, Op0, Op1);
12254 default: return Register();
12255 }
12256}
12257
12258// FastEmit functions for ISD::SMAX.
12259
12260Register fastEmit_ISD_SMAX_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12261 if (RetVT.SimpleTy != MVT::i32)
12262 return Register();
12263 if ((Subtarget->hasCSSC())) {
12264 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12265 }
12266 return Register();
12267}
12268
12269Register fastEmit_ISD_SMAX_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12270 if (RetVT.SimpleTy != MVT::i64)
12271 return Register();
12272 if ((Subtarget->hasCSSC())) {
12273 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12274 }
12275 return Register();
12276}
12277
12278Register fastEmit_ISD_SMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12279 if (RetVT.SimpleTy != MVT::v8i8)
12280 return Register();
12281 if ((Subtarget->isNeonAvailable())) {
12282 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12283 }
12284 return Register();
12285}
12286
12287Register fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12288 if (RetVT.SimpleTy != MVT::v16i8)
12289 return Register();
12290 if ((Subtarget->isNeonAvailable())) {
12291 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12292 }
12293 return Register();
12294}
12295
12296Register fastEmit_ISD_SMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12297 if (RetVT.SimpleTy != MVT::v4i16)
12298 return Register();
12299 if ((Subtarget->isNeonAvailable())) {
12300 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12301 }
12302 return Register();
12303}
12304
12305Register fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12306 if (RetVT.SimpleTy != MVT::v8i16)
12307 return Register();
12308 if ((Subtarget->isNeonAvailable())) {
12309 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12310 }
12311 return Register();
12312}
12313
12314Register fastEmit_ISD_SMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12315 if (RetVT.SimpleTy != MVT::v2i32)
12316 return Register();
12317 if ((Subtarget->isNeonAvailable())) {
12318 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12319 }
12320 return Register();
12321}
12322
12323Register fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12324 if (RetVT.SimpleTy != MVT::v4i32)
12325 return Register();
12326 if ((Subtarget->isNeonAvailable())) {
12327 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12328 }
12329 return Register();
12330}
12331
12332Register fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12333 switch (VT.SimpleTy) {
12334 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_rr(RetVT, Op0, Op1);
12335 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_rr(RetVT, Op0, Op1);
12336 case MVT::v8i8: return fastEmit_ISD_SMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
12337 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
12338 case MVT::v4i16: return fastEmit_ISD_SMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
12339 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
12340 case MVT::v2i32: return fastEmit_ISD_SMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
12341 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
12342 default: return Register();
12343 }
12344}
12345
12346// FastEmit functions for ISD::SMIN.
12347
12348Register fastEmit_ISD_SMIN_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12349 if (RetVT.SimpleTy != MVT::i32)
12350 return Register();
12351 if ((Subtarget->hasCSSC())) {
12352 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12353 }
12354 return Register();
12355}
12356
12357Register fastEmit_ISD_SMIN_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12358 if (RetVT.SimpleTy != MVT::i64)
12359 return Register();
12360 if ((Subtarget->hasCSSC())) {
12361 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12362 }
12363 return Register();
12364}
12365
12366Register fastEmit_ISD_SMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12367 if (RetVT.SimpleTy != MVT::v8i8)
12368 return Register();
12369 if ((Subtarget->isNeonAvailable())) {
12370 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12371 }
12372 return Register();
12373}
12374
12375Register fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12376 if (RetVT.SimpleTy != MVT::v16i8)
12377 return Register();
12378 if ((Subtarget->isNeonAvailable())) {
12379 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12380 }
12381 return Register();
12382}
12383
12384Register fastEmit_ISD_SMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12385 if (RetVT.SimpleTy != MVT::v4i16)
12386 return Register();
12387 if ((Subtarget->isNeonAvailable())) {
12388 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12389 }
12390 return Register();
12391}
12392
12393Register fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12394 if (RetVT.SimpleTy != MVT::v8i16)
12395 return Register();
12396 if ((Subtarget->isNeonAvailable())) {
12397 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12398 }
12399 return Register();
12400}
12401
12402Register fastEmit_ISD_SMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12403 if (RetVT.SimpleTy != MVT::v2i32)
12404 return Register();
12405 if ((Subtarget->isNeonAvailable())) {
12406 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12407 }
12408 return Register();
12409}
12410
12411Register fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12412 if (RetVT.SimpleTy != MVT::v4i32)
12413 return Register();
12414 if ((Subtarget->isNeonAvailable())) {
12415 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12416 }
12417 return Register();
12418}
12419
12420Register fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12421 switch (VT.SimpleTy) {
12422 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_rr(RetVT, Op0, Op1);
12423 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_rr(RetVT, Op0, Op1);
12424 case MVT::v8i8: return fastEmit_ISD_SMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
12425 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
12426 case MVT::v4i16: return fastEmit_ISD_SMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
12427 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
12428 case MVT::v2i32: return fastEmit_ISD_SMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
12429 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
12430 default: return Register();
12431 }
12432}
12433
12434// FastEmit functions for ISD::SRA.
12435
12436Register fastEmit_ISD_SRA_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12437 if (RetVT.SimpleTy != MVT::i64)
12438 return Register();
12439 return fastEmitInst_rr(MachineInstOpcode: AArch64::ASRVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12440}
12441
12442Register fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12443 switch (VT.SimpleTy) {
12444 case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_rr(RetVT, Op0, Op1);
12445 default: return Register();
12446 }
12447}
12448
12449// FastEmit functions for ISD::SRL.
12450
12451Register fastEmit_ISD_SRL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12452 if (RetVT.SimpleTy != MVT::i64)
12453 return Register();
12454 return fastEmitInst_rr(MachineInstOpcode: AArch64::LSRVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12455}
12456
12457Register fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12458 switch (VT.SimpleTy) {
12459 case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_rr(RetVT, Op0, Op1);
12460 default: return Register();
12461 }
12462}
12463
12464// FastEmit functions for ISD::SSUBSAT.
12465
12466Register fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12467 if (RetVT.SimpleTy != MVT::v8i8)
12468 return Register();
12469 if ((Subtarget->isNeonAvailable())) {
12470 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12471 }
12472 return Register();
12473}
12474
12475Register fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12476 if (RetVT.SimpleTy != MVT::v16i8)
12477 return Register();
12478 if ((Subtarget->isNeonAvailable())) {
12479 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12480 }
12481 return Register();
12482}
12483
12484Register fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12485 if (RetVT.SimpleTy != MVT::v4i16)
12486 return Register();
12487 if ((Subtarget->isNeonAvailable())) {
12488 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12489 }
12490 return Register();
12491}
12492
12493Register fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12494 if (RetVT.SimpleTy != MVT::v8i16)
12495 return Register();
12496 if ((Subtarget->isNeonAvailable())) {
12497 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12498 }
12499 return Register();
12500}
12501
12502Register fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12503 if (RetVT.SimpleTy != MVT::v2i32)
12504 return Register();
12505 if ((Subtarget->isNeonAvailable())) {
12506 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12507 }
12508 return Register();
12509}
12510
12511Register fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12512 if (RetVT.SimpleTy != MVT::v4i32)
12513 return Register();
12514 if ((Subtarget->isNeonAvailable())) {
12515 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12516 }
12517 return Register();
12518}
12519
12520Register fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12521 if (RetVT.SimpleTy != MVT::v1i64)
12522 return Register();
12523 if ((Subtarget->isNeonAvailable())) {
12524 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
12525 }
12526 return Register();
12527}
12528
12529Register fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12530 if (RetVT.SimpleTy != MVT::v2i64)
12531 return Register();
12532 if ((Subtarget->isNeonAvailable())) {
12533 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12534 }
12535 return Register();
12536}
12537
12538Register fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12539 if (RetVT.SimpleTy != MVT::nxv16i8)
12540 return Register();
12541 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12542 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
12543 }
12544 return Register();
12545}
12546
12547Register fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12548 if (RetVT.SimpleTy != MVT::nxv8i16)
12549 return Register();
12550 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12551 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
12552 }
12553 return Register();
12554}
12555
12556Register fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12557 if (RetVT.SimpleTy != MVT::nxv4i32)
12558 return Register();
12559 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12560 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
12561 }
12562 return Register();
12563}
12564
12565Register fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12566 if (RetVT.SimpleTy != MVT::nxv2i64)
12567 return Register();
12568 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12569 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
12570 }
12571 return Register();
12572}
12573
12574Register fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12575 switch (VT.SimpleTy) {
12576 case MVT::v8i8: return fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
12577 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12578 case MVT::v4i16: return fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
12579 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12580 case MVT::v2i32: return fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
12581 case MVT::v4i32: return fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
12582 case MVT::v1i64: return fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
12583 case MVT::v2i64: return fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
12584 case MVT::nxv16i8: return fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12585 case MVT::nxv8i16: return fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12586 case MVT::nxv4i32: return fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12587 case MVT::nxv2i64: return fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12588 default: return Register();
12589 }
12590}
12591
12592// FastEmit functions for ISD::STRICT_FADD.
12593
12594Register fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12595 if (RetVT.SimpleTy != MVT::f16)
12596 return Register();
12597 if ((Subtarget->hasFullFP16())) {
12598 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12599 }
12600 return Register();
12601}
12602
12603Register fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12604 if (RetVT.SimpleTy != MVT::f32)
12605 return Register();
12606 if ((Subtarget->hasFPARMv8())) {
12607 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12608 }
12609 return Register();
12610}
12611
12612Register fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12613 if (RetVT.SimpleTy != MVT::f64)
12614 return Register();
12615 if ((Subtarget->hasFPARMv8())) {
12616 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12617 }
12618 return Register();
12619}
12620
12621Register fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12622 if (RetVT.SimpleTy != MVT::v4f16)
12623 return Register();
12624 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12625 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12626 }
12627 return Register();
12628}
12629
12630Register fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12631 if (RetVT.SimpleTy != MVT::v8f16)
12632 return Register();
12633 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12634 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12635 }
12636 return Register();
12637}
12638
12639Register fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12640 if (RetVT.SimpleTy != MVT::v2f32)
12641 return Register();
12642 if ((Subtarget->isNeonAvailable())) {
12643 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12644 }
12645 return Register();
12646}
12647
12648Register fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12649 if (RetVT.SimpleTy != MVT::v4f32)
12650 return Register();
12651 if ((Subtarget->isNeonAvailable())) {
12652 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12653 }
12654 return Register();
12655}
12656
12657Register fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12658 if (RetVT.SimpleTy != MVT::v2f64)
12659 return Register();
12660 if ((Subtarget->isNeonAvailable())) {
12661 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12662 }
12663 return Register();
12664}
12665
12666Register fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12667 switch (VT.SimpleTy) {
12668 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1);
12669 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1);
12670 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1);
12671 case MVT::v4f16: return fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
12672 case MVT::v8f16: return fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
12673 case MVT::v2f32: return fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
12674 case MVT::v4f32: return fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
12675 case MVT::v2f64: return fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
12676 default: return Register();
12677 }
12678}
12679
12680// FastEmit functions for ISD::STRICT_FDIV.
12681
12682Register fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12683 if (RetVT.SimpleTy != MVT::f16)
12684 return Register();
12685 if ((Subtarget->hasFullFP16())) {
12686 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12687 }
12688 return Register();
12689}
12690
12691Register fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12692 if (RetVT.SimpleTy != MVT::f32)
12693 return Register();
12694 if ((Subtarget->hasFPARMv8())) {
12695 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12696 }
12697 return Register();
12698}
12699
12700Register fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12701 if (RetVT.SimpleTy != MVT::f64)
12702 return Register();
12703 if ((Subtarget->hasFPARMv8())) {
12704 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12705 }
12706 return Register();
12707}
12708
12709Register fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12710 if (RetVT.SimpleTy != MVT::v4f16)
12711 return Register();
12712 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12713 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12714 }
12715 return Register();
12716}
12717
12718Register fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12719 if (RetVT.SimpleTy != MVT::v8f16)
12720 return Register();
12721 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12722 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12723 }
12724 return Register();
12725}
12726
12727Register fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12728 if (RetVT.SimpleTy != MVT::v2f32)
12729 return Register();
12730 if ((Subtarget->isNeonAvailable())) {
12731 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12732 }
12733 return Register();
12734}
12735
12736Register fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12737 if (RetVT.SimpleTy != MVT::v4f32)
12738 return Register();
12739 if ((Subtarget->isNeonAvailable())) {
12740 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12741 }
12742 return Register();
12743}
12744
12745Register fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12746 if (RetVT.SimpleTy != MVT::v2f64)
12747 return Register();
12748 if ((Subtarget->isNeonAvailable())) {
12749 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12750 }
12751 return Register();
12752}
12753
12754Register fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12755 switch (VT.SimpleTy) {
12756 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
12757 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
12758 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
12759 case MVT::v4f16: return fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1);
12760 case MVT::v8f16: return fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
12761 case MVT::v2f32: return fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1);
12762 case MVT::v4f32: return fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
12763 case MVT::v2f64: return fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
12764 default: return Register();
12765 }
12766}
12767
12768// FastEmit functions for ISD::STRICT_FMAXIMUM.
12769
12770Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12771 if (RetVT.SimpleTy != MVT::f16)
12772 return Register();
12773 if ((Subtarget->hasFullFP16())) {
12774 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12775 }
12776 return Register();
12777}
12778
12779Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12780 if (RetVT.SimpleTy != MVT::f32)
12781 return Register();
12782 if ((Subtarget->hasFPARMv8())) {
12783 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12784 }
12785 return Register();
12786}
12787
12788Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12789 if (RetVT.SimpleTy != MVT::f64)
12790 return Register();
12791 if ((Subtarget->hasFPARMv8())) {
12792 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12793 }
12794 return Register();
12795}
12796
12797Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12798 if (RetVT.SimpleTy != MVT::v4f16)
12799 return Register();
12800 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12801 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12802 }
12803 return Register();
12804}
12805
12806Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12807 if (RetVT.SimpleTy != MVT::v8f16)
12808 return Register();
12809 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12810 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12811 }
12812 return Register();
12813}
12814
12815Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12816 if (RetVT.SimpleTy != MVT::v2f32)
12817 return Register();
12818 if ((Subtarget->isNeonAvailable())) {
12819 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12820 }
12821 return Register();
12822}
12823
12824Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12825 if (RetVT.SimpleTy != MVT::v4f32)
12826 return Register();
12827 if ((Subtarget->isNeonAvailable())) {
12828 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12829 }
12830 return Register();
12831}
12832
12833Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12834 if (RetVT.SimpleTy != MVT::v2f64)
12835 return Register();
12836 if ((Subtarget->isNeonAvailable())) {
12837 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12838 }
12839 return Register();
12840}
12841
12842Register fastEmit_ISD_STRICT_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12843 switch (VT.SimpleTy) {
12844 case MVT::f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1);
12845 case MVT::f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1);
12846 case MVT::f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1);
12847 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
12848 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
12849 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
12850 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
12851 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
12852 default: return Register();
12853 }
12854}
12855
12856// FastEmit functions for ISD::STRICT_FMAXNUM.
12857
12858Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12859 if (RetVT.SimpleTy != MVT::f16)
12860 return Register();
12861 if ((Subtarget->hasFullFP16())) {
12862 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12863 }
12864 return Register();
12865}
12866
12867Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12868 if (RetVT.SimpleTy != MVT::f32)
12869 return Register();
12870 if ((Subtarget->hasFPARMv8())) {
12871 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12872 }
12873 return Register();
12874}
12875
12876Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12877 if (RetVT.SimpleTy != MVT::f64)
12878 return Register();
12879 if ((Subtarget->hasFPARMv8())) {
12880 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12881 }
12882 return Register();
12883}
12884
12885Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12886 if (RetVT.SimpleTy != MVT::v4f16)
12887 return Register();
12888 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12889 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12890 }
12891 return Register();
12892}
12893
12894Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12895 if (RetVT.SimpleTy != MVT::v8f16)
12896 return Register();
12897 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12898 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12899 }
12900 return Register();
12901}
12902
12903Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12904 if (RetVT.SimpleTy != MVT::v2f32)
12905 return Register();
12906 if ((Subtarget->isNeonAvailable())) {
12907 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12908 }
12909 return Register();
12910}
12911
12912Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12913 if (RetVT.SimpleTy != MVT::v4f32)
12914 return Register();
12915 if ((Subtarget->isNeonAvailable())) {
12916 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12917 }
12918 return Register();
12919}
12920
12921Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12922 if (RetVT.SimpleTy != MVT::v2f64)
12923 return Register();
12924 if ((Subtarget->isNeonAvailable())) {
12925 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12926 }
12927 return Register();
12928}
12929
12930Register fastEmit_ISD_STRICT_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12931 switch (VT.SimpleTy) {
12932 case MVT::f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
12933 case MVT::f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
12934 case MVT::f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
12935 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
12936 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
12937 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
12938 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
12939 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
12940 default: return Register();
12941 }
12942}
12943
12944// FastEmit functions for ISD::STRICT_FMINIMUM.
12945
12946Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12947 if (RetVT.SimpleTy != MVT::f16)
12948 return Register();
12949 if ((Subtarget->hasFullFP16())) {
12950 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12951 }
12952 return Register();
12953}
12954
12955Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12956 if (RetVT.SimpleTy != MVT::f32)
12957 return Register();
12958 if ((Subtarget->hasFPARMv8())) {
12959 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12960 }
12961 return Register();
12962}
12963
12964Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12965 if (RetVT.SimpleTy != MVT::f64)
12966 return Register();
12967 if ((Subtarget->hasFPARMv8())) {
12968 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12969 }
12970 return Register();
12971}
12972
12973Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12974 if (RetVT.SimpleTy != MVT::v4f16)
12975 return Register();
12976 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12977 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12978 }
12979 return Register();
12980}
12981
12982Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12983 if (RetVT.SimpleTy != MVT::v8f16)
12984 return Register();
12985 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12986 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12987 }
12988 return Register();
12989}
12990
12991Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12992 if (RetVT.SimpleTy != MVT::v2f32)
12993 return Register();
12994 if ((Subtarget->isNeonAvailable())) {
12995 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12996 }
12997 return Register();
12998}
12999
13000Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13001 if (RetVT.SimpleTy != MVT::v4f32)
13002 return Register();
13003 if ((Subtarget->isNeonAvailable())) {
13004 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13005 }
13006 return Register();
13007}
13008
13009Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13010 if (RetVT.SimpleTy != MVT::v2f64)
13011 return Register();
13012 if ((Subtarget->isNeonAvailable())) {
13013 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13014 }
13015 return Register();
13016}
13017
13018Register fastEmit_ISD_STRICT_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13019 switch (VT.SimpleTy) {
13020 case MVT::f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1);
13021 case MVT::f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1);
13022 case MVT::f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1);
13023 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13024 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13025 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13026 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13027 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13028 default: return Register();
13029 }
13030}
13031
13032// FastEmit functions for ISD::STRICT_FMINNUM.
13033
13034Register fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13035 if (RetVT.SimpleTy != MVT::f16)
13036 return Register();
13037 if ((Subtarget->hasFullFP16())) {
13038 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13039 }
13040 return Register();
13041}
13042
13043Register fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13044 if (RetVT.SimpleTy != MVT::f32)
13045 return Register();
13046 if ((Subtarget->hasFPARMv8())) {
13047 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13048 }
13049 return Register();
13050}
13051
13052Register fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13053 if (RetVT.SimpleTy != MVT::f64)
13054 return Register();
13055 if ((Subtarget->hasFPARMv8())) {
13056 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13057 }
13058 return Register();
13059}
13060
13061Register fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13062 if (RetVT.SimpleTy != MVT::v4f16)
13063 return Register();
13064 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13065 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13066 }
13067 return Register();
13068}
13069
13070Register fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13071 if (RetVT.SimpleTy != MVT::v8f16)
13072 return Register();
13073 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13074 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13075 }
13076 return Register();
13077}
13078
13079Register fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13080 if (RetVT.SimpleTy != MVT::v2f32)
13081 return Register();
13082 if ((Subtarget->isNeonAvailable())) {
13083 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13084 }
13085 return Register();
13086}
13087
13088Register fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13089 if (RetVT.SimpleTy != MVT::v4f32)
13090 return Register();
13091 if ((Subtarget->isNeonAvailable())) {
13092 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13093 }
13094 return Register();
13095}
13096
13097Register fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13098 if (RetVT.SimpleTy != MVT::v2f64)
13099 return Register();
13100 if ((Subtarget->isNeonAvailable())) {
13101 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13102 }
13103 return Register();
13104}
13105
13106Register fastEmit_ISD_STRICT_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13107 switch (VT.SimpleTy) {
13108 case MVT::f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
13109 case MVT::f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
13110 case MVT::f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
13111 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13112 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13113 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13114 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13115 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13116 default: return Register();
13117 }
13118}
13119
13120// FastEmit functions for ISD::STRICT_FMUL.
13121
13122Register fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13123 if (RetVT.SimpleTy != MVT::f16)
13124 return Register();
13125 if ((Subtarget->hasFullFP16())) {
13126 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13127 }
13128 return Register();
13129}
13130
13131Register fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13132 if (RetVT.SimpleTy != MVT::f32)
13133 return Register();
13134 if ((Subtarget->hasFPARMv8())) {
13135 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13136 }
13137 return Register();
13138}
13139
13140Register fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13141 if (RetVT.SimpleTy != MVT::f64)
13142 return Register();
13143 if ((Subtarget->hasFPARMv8())) {
13144 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13145 }
13146 return Register();
13147}
13148
13149Register fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13150 if (RetVT.SimpleTy != MVT::v4f16)
13151 return Register();
13152 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13153 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13154 }
13155 return Register();
13156}
13157
13158Register fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13159 if (RetVT.SimpleTy != MVT::v8f16)
13160 return Register();
13161 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13162 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13163 }
13164 return Register();
13165}
13166
13167Register fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13168 if (RetVT.SimpleTy != MVT::v2f32)
13169 return Register();
13170 if ((Subtarget->isNeonAvailable())) {
13171 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13172 }
13173 return Register();
13174}
13175
13176Register fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13177 if (RetVT.SimpleTy != MVT::v4f32)
13178 return Register();
13179 if ((Subtarget->isNeonAvailable())) {
13180 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13181 }
13182 return Register();
13183}
13184
13185Register fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13186 if (RetVT.SimpleTy != MVT::v2f64)
13187 return Register();
13188 if ((Subtarget->isNeonAvailable())) {
13189 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13190 }
13191 return Register();
13192}
13193
13194Register fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13195 switch (VT.SimpleTy) {
13196 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
13197 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
13198 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
13199 case MVT::v4f16: return fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
13200 case MVT::v8f16: return fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
13201 case MVT::v2f32: return fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
13202 case MVT::v4f32: return fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
13203 case MVT::v2f64: return fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
13204 default: return Register();
13205 }
13206}
13207
13208// FastEmit functions for ISD::STRICT_FSUB.
13209
13210Register fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13211 if (RetVT.SimpleTy != MVT::f16)
13212 return Register();
13213 if ((Subtarget->hasFullFP16())) {
13214 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13215 }
13216 return Register();
13217}
13218
13219Register fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13220 if (RetVT.SimpleTy != MVT::f32)
13221 return Register();
13222 if ((Subtarget->hasFPARMv8())) {
13223 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13224 }
13225 return Register();
13226}
13227
13228Register fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13229 if (RetVT.SimpleTy != MVT::f64)
13230 return Register();
13231 if ((Subtarget->hasFPARMv8())) {
13232 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13233 }
13234 return Register();
13235}
13236
13237Register fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13238 if (RetVT.SimpleTy != MVT::v4f16)
13239 return Register();
13240 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13241 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13242 }
13243 return Register();
13244}
13245
13246Register fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13247 if (RetVT.SimpleTy != MVT::v8f16)
13248 return Register();
13249 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13250 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13251 }
13252 return Register();
13253}
13254
13255Register fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13256 if (RetVT.SimpleTy != MVT::v2f32)
13257 return Register();
13258 if ((Subtarget->isNeonAvailable())) {
13259 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13260 }
13261 return Register();
13262}
13263
13264Register fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13265 if (RetVT.SimpleTy != MVT::v4f32)
13266 return Register();
13267 if ((Subtarget->isNeonAvailable())) {
13268 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13269 }
13270 return Register();
13271}
13272
13273Register fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13274 if (RetVT.SimpleTy != MVT::v2f64)
13275 return Register();
13276 if ((Subtarget->isNeonAvailable())) {
13277 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13278 }
13279 return Register();
13280}
13281
13282Register fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13283 switch (VT.SimpleTy) {
13284 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
13285 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
13286 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
13287 case MVT::v4f16: return fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
13288 case MVT::v8f16: return fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
13289 case MVT::v2f32: return fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
13290 case MVT::v4f32: return fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
13291 case MVT::v2f64: return fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
13292 default: return Register();
13293 }
13294}
13295
13296// FastEmit functions for ISD::SUB.
13297
13298Register fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13299 if (RetVT.SimpleTy != MVT::i32)
13300 return Register();
13301 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBSWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13302}
13303
13304Register fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13305 if (RetVT.SimpleTy != MVT::i64)
13306 return Register();
13307 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBSXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13308}
13309
13310Register fastEmit_ISD_SUB_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13311 if (RetVT.SimpleTy != MVT::v8i8)
13312 return Register();
13313 if ((Subtarget->isNeonAvailable())) {
13314 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13315 }
13316 return Register();
13317}
13318
13319Register fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13320 if (RetVT.SimpleTy != MVT::v16i8)
13321 return Register();
13322 if ((Subtarget->isNeonAvailable())) {
13323 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13324 }
13325 return Register();
13326}
13327
13328Register fastEmit_ISD_SUB_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13329 if (RetVT.SimpleTy != MVT::v4i16)
13330 return Register();
13331 if ((Subtarget->isNeonAvailable())) {
13332 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13333 }
13334 return Register();
13335}
13336
13337Register fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13338 if (RetVT.SimpleTy != MVT::v8i16)
13339 return Register();
13340 if ((Subtarget->isNeonAvailable())) {
13341 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13342 }
13343 return Register();
13344}
13345
13346Register fastEmit_ISD_SUB_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13347 if (RetVT.SimpleTy != MVT::v2i32)
13348 return Register();
13349 if ((Subtarget->isNeonAvailable())) {
13350 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13351 }
13352 return Register();
13353}
13354
13355Register fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13356 if (RetVT.SimpleTy != MVT::v4i32)
13357 return Register();
13358 if ((Subtarget->isNeonAvailable())) {
13359 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13360 }
13361 return Register();
13362}
13363
13364Register fastEmit_ISD_SUB_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13365 if (RetVT.SimpleTy != MVT::v1i64)
13366 return Register();
13367 if ((Subtarget->isNeonAvailable())) {
13368 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13369 }
13370 return Register();
13371}
13372
13373Register fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13374 if (RetVT.SimpleTy != MVT::v2i64)
13375 return Register();
13376 if ((Subtarget->isNeonAvailable())) {
13377 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13378 }
13379 return Register();
13380}
13381
13382Register fastEmit_ISD_SUB_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13383 if (RetVT.SimpleTy != MVT::nxv16i8)
13384 return Register();
13385 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13386 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13387 }
13388 return Register();
13389}
13390
13391Register fastEmit_ISD_SUB_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13392 if (RetVT.SimpleTy != MVT::nxv8i16)
13393 return Register();
13394 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13395 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13396 }
13397 return Register();
13398}
13399
13400Register fastEmit_ISD_SUB_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13401 if (RetVT.SimpleTy != MVT::nxv4i32)
13402 return Register();
13403 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13404 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13405 }
13406 return Register();
13407}
13408
13409Register fastEmit_ISD_SUB_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13410 if (RetVT.SimpleTy != MVT::nxv2i64)
13411 return Register();
13412 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13413 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13414 }
13415 return Register();
13416}
13417
13418Register fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13419 switch (VT.SimpleTy) {
13420 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
13421 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op1);
13422 case MVT::v8i8: return fastEmit_ISD_SUB_MVT_v8i8_rr(RetVT, Op0, Op1);
13423 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
13424 case MVT::v4i16: return fastEmit_ISD_SUB_MVT_v4i16_rr(RetVT, Op0, Op1);
13425 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
13426 case MVT::v2i32: return fastEmit_ISD_SUB_MVT_v2i32_rr(RetVT, Op0, Op1);
13427 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
13428 case MVT::v1i64: return fastEmit_ISD_SUB_MVT_v1i64_rr(RetVT, Op0, Op1);
13429 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
13430 case MVT::nxv16i8: return fastEmit_ISD_SUB_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13431 case MVT::nxv8i16: return fastEmit_ISD_SUB_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13432 case MVT::nxv4i32: return fastEmit_ISD_SUB_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13433 case MVT::nxv2i64: return fastEmit_ISD_SUB_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13434 default: return Register();
13435 }
13436}
13437
13438// FastEmit functions for ISD::UADDSAT.
13439
13440Register fastEmit_ISD_UADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13441 if (RetVT.SimpleTy != MVT::v8i8)
13442 return Register();
13443 if ((Subtarget->isNeonAvailable())) {
13444 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13445 }
13446 return Register();
13447}
13448
13449Register fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13450 if (RetVT.SimpleTy != MVT::v16i8)
13451 return Register();
13452 if ((Subtarget->isNeonAvailable())) {
13453 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13454 }
13455 return Register();
13456}
13457
13458Register fastEmit_ISD_UADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13459 if (RetVT.SimpleTy != MVT::v4i16)
13460 return Register();
13461 if ((Subtarget->isNeonAvailable())) {
13462 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13463 }
13464 return Register();
13465}
13466
13467Register fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13468 if (RetVT.SimpleTy != MVT::v8i16)
13469 return Register();
13470 if ((Subtarget->isNeonAvailable())) {
13471 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13472 }
13473 return Register();
13474}
13475
13476Register fastEmit_ISD_UADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13477 if (RetVT.SimpleTy != MVT::v2i32)
13478 return Register();
13479 if ((Subtarget->isNeonAvailable())) {
13480 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13481 }
13482 return Register();
13483}
13484
13485Register fastEmit_ISD_UADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13486 if (RetVT.SimpleTy != MVT::v4i32)
13487 return Register();
13488 if ((Subtarget->isNeonAvailable())) {
13489 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13490 }
13491 return Register();
13492}
13493
13494Register fastEmit_ISD_UADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13495 if (RetVT.SimpleTy != MVT::v1i64)
13496 return Register();
13497 if ((Subtarget->isNeonAvailable())) {
13498 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13499 }
13500 return Register();
13501}
13502
13503Register fastEmit_ISD_UADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13504 if (RetVT.SimpleTy != MVT::v2i64)
13505 return Register();
13506 if ((Subtarget->isNeonAvailable())) {
13507 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13508 }
13509 return Register();
13510}
13511
13512Register fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13513 if (RetVT.SimpleTy != MVT::nxv16i8)
13514 return Register();
13515 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13516 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13517 }
13518 return Register();
13519}
13520
13521Register fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13522 if (RetVT.SimpleTy != MVT::nxv8i16)
13523 return Register();
13524 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13525 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13526 }
13527 return Register();
13528}
13529
13530Register fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13531 if (RetVT.SimpleTy != MVT::nxv4i32)
13532 return Register();
13533 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13534 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13535 }
13536 return Register();
13537}
13538
13539Register fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13540 if (RetVT.SimpleTy != MVT::nxv2i64)
13541 return Register();
13542 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13543 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13544 }
13545 return Register();
13546}
13547
13548Register fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13549 switch (VT.SimpleTy) {
13550 case MVT::v8i8: return fastEmit_ISD_UADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
13551 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13552 case MVT::v4i16: return fastEmit_ISD_UADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
13553 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13554 case MVT::v2i32: return fastEmit_ISD_UADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
13555 case MVT::v4i32: return fastEmit_ISD_UADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
13556 case MVT::v1i64: return fastEmit_ISD_UADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
13557 case MVT::v2i64: return fastEmit_ISD_UADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
13558 case MVT::nxv16i8: return fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13559 case MVT::nxv8i16: return fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13560 case MVT::nxv4i32: return fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13561 case MVT::nxv2i64: return fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13562 default: return Register();
13563 }
13564}
13565
13566// FastEmit functions for ISD::UDIV.
13567
13568Register fastEmit_ISD_UDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13569 if (RetVT.SimpleTy != MVT::i32)
13570 return Register();
13571 return fastEmitInst_rr(MachineInstOpcode: AArch64::UDIVWr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13572}
13573
13574Register fastEmit_ISD_UDIV_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13575 if (RetVT.SimpleTy != MVT::i64)
13576 return Register();
13577 return fastEmitInst_rr(MachineInstOpcode: AArch64::UDIVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13578}
13579
13580Register fastEmit_ISD_UDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13581 switch (VT.SimpleTy) {
13582 case MVT::i32: return fastEmit_ISD_UDIV_MVT_i32_rr(RetVT, Op0, Op1);
13583 case MVT::i64: return fastEmit_ISD_UDIV_MVT_i64_rr(RetVT, Op0, Op1);
13584 default: return Register();
13585 }
13586}
13587
13588// FastEmit functions for ISD::UMAX.
13589
13590Register fastEmit_ISD_UMAX_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13591 if (RetVT.SimpleTy != MVT::i32)
13592 return Register();
13593 if ((Subtarget->hasCSSC())) {
13594 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13595 }
13596 return Register();
13597}
13598
13599Register fastEmit_ISD_UMAX_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13600 if (RetVT.SimpleTy != MVT::i64)
13601 return Register();
13602 if ((Subtarget->hasCSSC())) {
13603 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13604 }
13605 return Register();
13606}
13607
13608Register fastEmit_ISD_UMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13609 if (RetVT.SimpleTy != MVT::v8i8)
13610 return Register();
13611 if ((Subtarget->isNeonAvailable())) {
13612 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13613 }
13614 return Register();
13615}
13616
13617Register fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13618 if (RetVT.SimpleTy != MVT::v16i8)
13619 return Register();
13620 if ((Subtarget->isNeonAvailable())) {
13621 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13622 }
13623 return Register();
13624}
13625
13626Register fastEmit_ISD_UMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13627 if (RetVT.SimpleTy != MVT::v4i16)
13628 return Register();
13629 if ((Subtarget->isNeonAvailable())) {
13630 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13631 }
13632 return Register();
13633}
13634
13635Register fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13636 if (RetVT.SimpleTy != MVT::v8i16)
13637 return Register();
13638 if ((Subtarget->isNeonAvailable())) {
13639 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13640 }
13641 return Register();
13642}
13643
13644Register fastEmit_ISD_UMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13645 if (RetVT.SimpleTy != MVT::v2i32)
13646 return Register();
13647 if ((Subtarget->isNeonAvailable())) {
13648 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13649 }
13650 return Register();
13651}
13652
13653Register fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13654 if (RetVT.SimpleTy != MVT::v4i32)
13655 return Register();
13656 if ((Subtarget->isNeonAvailable())) {
13657 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13658 }
13659 return Register();
13660}
13661
13662Register fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13663 switch (VT.SimpleTy) {
13664 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_rr(RetVT, Op0, Op1);
13665 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_rr(RetVT, Op0, Op1);
13666 case MVT::v8i8: return fastEmit_ISD_UMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
13667 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
13668 case MVT::v4i16: return fastEmit_ISD_UMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
13669 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
13670 case MVT::v2i32: return fastEmit_ISD_UMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
13671 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
13672 default: return Register();
13673 }
13674}
13675
13676// FastEmit functions for ISD::UMIN.
13677
13678Register fastEmit_ISD_UMIN_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13679 if (RetVT.SimpleTy != MVT::i32)
13680 return Register();
13681 if ((Subtarget->hasCSSC())) {
13682 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13683 }
13684 return Register();
13685}
13686
13687Register fastEmit_ISD_UMIN_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13688 if (RetVT.SimpleTy != MVT::i64)
13689 return Register();
13690 if ((Subtarget->hasCSSC())) {
13691 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13692 }
13693 return Register();
13694}
13695
13696Register fastEmit_ISD_UMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13697 if (RetVT.SimpleTy != MVT::v8i8)
13698 return Register();
13699 if ((Subtarget->isNeonAvailable())) {
13700 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13701 }
13702 return Register();
13703}
13704
13705Register fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13706 if (RetVT.SimpleTy != MVT::v16i8)
13707 return Register();
13708 if ((Subtarget->isNeonAvailable())) {
13709 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13710 }
13711 return Register();
13712}
13713
13714Register fastEmit_ISD_UMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13715 if (RetVT.SimpleTy != MVT::v4i16)
13716 return Register();
13717 if ((Subtarget->isNeonAvailable())) {
13718 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13719 }
13720 return Register();
13721}
13722
13723Register fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13724 if (RetVT.SimpleTy != MVT::v8i16)
13725 return Register();
13726 if ((Subtarget->isNeonAvailable())) {
13727 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13728 }
13729 return Register();
13730}
13731
13732Register fastEmit_ISD_UMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13733 if (RetVT.SimpleTy != MVT::v2i32)
13734 return Register();
13735 if ((Subtarget->isNeonAvailable())) {
13736 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13737 }
13738 return Register();
13739}
13740
13741Register fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13742 if (RetVT.SimpleTy != MVT::v4i32)
13743 return Register();
13744 if ((Subtarget->isNeonAvailable())) {
13745 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13746 }
13747 return Register();
13748}
13749
13750Register fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13751 switch (VT.SimpleTy) {
13752 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_rr(RetVT, Op0, Op1);
13753 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_rr(RetVT, Op0, Op1);
13754 case MVT::v8i8: return fastEmit_ISD_UMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
13755 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
13756 case MVT::v4i16: return fastEmit_ISD_UMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
13757 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
13758 case MVT::v2i32: return fastEmit_ISD_UMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
13759 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
13760 default: return Register();
13761 }
13762}
13763
13764// FastEmit functions for ISD::USUBSAT.
13765
13766Register fastEmit_ISD_USUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13767 if (RetVT.SimpleTy != MVT::v8i8)
13768 return Register();
13769 if ((Subtarget->isNeonAvailable())) {
13770 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13771 }
13772 return Register();
13773}
13774
13775Register fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13776 if (RetVT.SimpleTy != MVT::v16i8)
13777 return Register();
13778 if ((Subtarget->isNeonAvailable())) {
13779 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13780 }
13781 return Register();
13782}
13783
13784Register fastEmit_ISD_USUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13785 if (RetVT.SimpleTy != MVT::v4i16)
13786 return Register();
13787 if ((Subtarget->isNeonAvailable())) {
13788 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13789 }
13790 return Register();
13791}
13792
13793Register fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13794 if (RetVT.SimpleTy != MVT::v8i16)
13795 return Register();
13796 if ((Subtarget->isNeonAvailable())) {
13797 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13798 }
13799 return Register();
13800}
13801
13802Register fastEmit_ISD_USUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13803 if (RetVT.SimpleTy != MVT::v2i32)
13804 return Register();
13805 if ((Subtarget->isNeonAvailable())) {
13806 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13807 }
13808 return Register();
13809}
13810
13811Register fastEmit_ISD_USUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13812 if (RetVT.SimpleTy != MVT::v4i32)
13813 return Register();
13814 if ((Subtarget->isNeonAvailable())) {
13815 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13816 }
13817 return Register();
13818}
13819
13820Register fastEmit_ISD_USUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13821 if (RetVT.SimpleTy != MVT::v1i64)
13822 return Register();
13823 if ((Subtarget->isNeonAvailable())) {
13824 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13825 }
13826 return Register();
13827}
13828
13829Register fastEmit_ISD_USUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13830 if (RetVT.SimpleTy != MVT::v2i64)
13831 return Register();
13832 if ((Subtarget->isNeonAvailable())) {
13833 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13834 }
13835 return Register();
13836}
13837
13838Register fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13839 if (RetVT.SimpleTy != MVT::nxv16i8)
13840 return Register();
13841 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13842 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13843 }
13844 return Register();
13845}
13846
13847Register fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13848 if (RetVT.SimpleTy != MVT::nxv8i16)
13849 return Register();
13850 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13851 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13852 }
13853 return Register();
13854}
13855
13856Register fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13857 if (RetVT.SimpleTy != MVT::nxv4i32)
13858 return Register();
13859 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13860 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13861 }
13862 return Register();
13863}
13864
13865Register fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13866 if (RetVT.SimpleTy != MVT::nxv2i64)
13867 return Register();
13868 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13869 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13870 }
13871 return Register();
13872}
13873
13874Register fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13875 switch (VT.SimpleTy) {
13876 case MVT::v8i8: return fastEmit_ISD_USUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
13877 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13878 case MVT::v4i16: return fastEmit_ISD_USUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
13879 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13880 case MVT::v2i32: return fastEmit_ISD_USUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
13881 case MVT::v4i32: return fastEmit_ISD_USUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
13882 case MVT::v1i64: return fastEmit_ISD_USUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
13883 case MVT::v2i64: return fastEmit_ISD_USUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
13884 case MVT::nxv16i8: return fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13885 case MVT::nxv8i16: return fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13886 case MVT::nxv4i32: return fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13887 case MVT::nxv2i64: return fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13888 default: return Register();
13889 }
13890}
13891
13892// FastEmit functions for ISD::XOR.
13893
13894Register fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13895 if (RetVT.SimpleTy != MVT::i32)
13896 return Register();
13897 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13898}
13899
13900Register fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13901 if (RetVT.SimpleTy != MVT::i64)
13902 return Register();
13903 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13904}
13905
13906Register fastEmit_ISD_XOR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13907 if (RetVT.SimpleTy != MVT::v8i8)
13908 return Register();
13909 if ((Subtarget->isNeonAvailable())) {
13910 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13911 }
13912 return Register();
13913}
13914
13915Register fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13916 if (RetVT.SimpleTy != MVT::v16i8)
13917 return Register();
13918 if ((Subtarget->isNeonAvailable())) {
13919 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13920 }
13921 return Register();
13922}
13923
13924Register fastEmit_ISD_XOR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13925 if (RetVT.SimpleTy != MVT::v4i16)
13926 return Register();
13927 if ((Subtarget->isNeonAvailable())) {
13928 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13929 }
13930 return Register();
13931}
13932
13933Register fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13934 if (RetVT.SimpleTy != MVT::v8i16)
13935 return Register();
13936 if ((Subtarget->isNeonAvailable())) {
13937 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13938 }
13939 return Register();
13940}
13941
13942Register fastEmit_ISD_XOR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13943 if (RetVT.SimpleTy != MVT::v2i32)
13944 return Register();
13945 if ((Subtarget->isNeonAvailable())) {
13946 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13947 }
13948 return Register();
13949}
13950
13951Register fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13952 if (RetVT.SimpleTy != MVT::v4i32)
13953 return Register();
13954 if ((Subtarget->isNeonAvailable())) {
13955 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13956 }
13957 return Register();
13958}
13959
13960Register fastEmit_ISD_XOR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13961 if (RetVT.SimpleTy != MVT::v1i64)
13962 return Register();
13963 if ((Subtarget->isNeonAvailable())) {
13964 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13965 }
13966 return Register();
13967}
13968
13969Register fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13970 if (RetVT.SimpleTy != MVT::v2i64)
13971 return Register();
13972 if ((Subtarget->isNeonAvailable())) {
13973 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13974 }
13975 return Register();
13976}
13977
13978Register fastEmit_ISD_XOR_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13979 if (RetVT.SimpleTy != MVT::nxv16i8)
13980 return Register();
13981 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13982 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
13983 }
13984 return Register();
13985}
13986
13987Register fastEmit_ISD_XOR_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13988 if (RetVT.SimpleTy != MVT::nxv8i16)
13989 return Register();
13990 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13991 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
13992 }
13993 return Register();
13994}
13995
13996Register fastEmit_ISD_XOR_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13997 if (RetVT.SimpleTy != MVT::nxv4i32)
13998 return Register();
13999 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14000 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14001 }
14002 return Register();
14003}
14004
14005Register fastEmit_ISD_XOR_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14006 if (RetVT.SimpleTy != MVT::nxv2i64)
14007 return Register();
14008 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14009 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14010 }
14011 return Register();
14012}
14013
14014Register fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14015 switch (VT.SimpleTy) {
14016 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
14017 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op1);
14018 case MVT::v8i8: return fastEmit_ISD_XOR_MVT_v8i8_rr(RetVT, Op0, Op1);
14019 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
14020 case MVT::v4i16: return fastEmit_ISD_XOR_MVT_v4i16_rr(RetVT, Op0, Op1);
14021 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
14022 case MVT::v2i32: return fastEmit_ISD_XOR_MVT_v2i32_rr(RetVT, Op0, Op1);
14023 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
14024 case MVT::v1i64: return fastEmit_ISD_XOR_MVT_v1i64_rr(RetVT, Op0, Op1);
14025 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
14026 case MVT::nxv16i8: return fastEmit_ISD_XOR_MVT_nxv16i8_rr(RetVT, Op0, Op1);
14027 case MVT::nxv8i16: return fastEmit_ISD_XOR_MVT_nxv8i16_rr(RetVT, Op0, Op1);
14028 case MVT::nxv4i32: return fastEmit_ISD_XOR_MVT_nxv4i32_rr(RetVT, Op0, Op1);
14029 case MVT::nxv2i64: return fastEmit_ISD_XOR_MVT_nxv2i64_rr(RetVT, Op0, Op1);
14030 default: return Register();
14031 }
14032}
14033
14034// Top-level FastEmit function.
14035
14036Register fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, Register Op1) override {
14037 switch (Opcode) {
14038 case AArch64ISD::ADDP: return fastEmit_AArch64ISD_ADDP_rr(VT, RetVT, Op0, Op1);
14039 case AArch64ISD::BIC: return fastEmit_AArch64ISD_BIC_rr(VT, RetVT, Op0, Op1);
14040 case AArch64ISD::FCMEQ: return fastEmit_AArch64ISD_FCMEQ_rr(VT, RetVT, Op0, Op1);
14041 case AArch64ISD::FCMGE: return fastEmit_AArch64ISD_FCMGE_rr(VT, RetVT, Op0, Op1);
14042 case AArch64ISD::FCMGT: return fastEmit_AArch64ISD_FCMGT_rr(VT, RetVT, Op0, Op1);
14043 case AArch64ISD::FCMP: return fastEmit_AArch64ISD_FCMP_rr(VT, RetVT, Op0, Op1);
14044 case AArch64ISD::FRECPS: return fastEmit_AArch64ISD_FRECPS_rr(VT, RetVT, Op0, Op1);
14045 case AArch64ISD::FRSQRTS: return fastEmit_AArch64ISD_FRSQRTS_rr(VT, RetVT, Op0, Op1);
14046 case AArch64ISD::PMULL: return fastEmit_AArch64ISD_PMULL_rr(VT, RetVT, Op0, Op1);
14047 case AArch64ISD::PTEST: return fastEmit_AArch64ISD_PTEST_rr(VT, RetVT, Op0, Op1);
14048 case AArch64ISD::PTEST_ANY: return fastEmit_AArch64ISD_PTEST_ANY_rr(VT, RetVT, Op0, Op1);
14049 case AArch64ISD::PTEST_FIRST: return fastEmit_AArch64ISD_PTEST_FIRST_rr(VT, RetVT, Op0, Op1);
14050 case AArch64ISD::SMULL: return fastEmit_AArch64ISD_SMULL_rr(VT, RetVT, Op0, Op1);
14051 case AArch64ISD::SQADD: return fastEmit_AArch64ISD_SQADD_rr(VT, RetVT, Op0, Op1);
14052 case AArch64ISD::SQDMULH: return fastEmit_AArch64ISD_SQDMULH_rr(VT, RetVT, Op0, Op1);
14053 case AArch64ISD::SQDMULL: return fastEmit_AArch64ISD_SQDMULL_rr(VT, RetVT, Op0, Op1);
14054 case AArch64ISD::SQRDMULH: return fastEmit_AArch64ISD_SQRDMULH_rr(VT, RetVT, Op0, Op1);
14055 case AArch64ISD::SQRSHL: return fastEmit_AArch64ISD_SQRSHL_rr(VT, RetVT, Op0, Op1);
14056 case AArch64ISD::SQSHL: return fastEmit_AArch64ISD_SQSHL_rr(VT, RetVT, Op0, Op1);
14057 case AArch64ISD::SQSUB: return fastEmit_AArch64ISD_SQSUB_rr(VT, RetVT, Op0, Op1);
14058 case AArch64ISD::STRICT_FCMP: return fastEmit_AArch64ISD_STRICT_FCMP_rr(VT, RetVT, Op0, Op1);
14059 case AArch64ISD::STRICT_FCMPE: return fastEmit_AArch64ISD_STRICT_FCMPE_rr(VT, RetVT, Op0, Op1);
14060 case AArch64ISD::SUQADD: return fastEmit_AArch64ISD_SUQADD_rr(VT, RetVT, Op0, Op1);
14061 case AArch64ISD::TBL: return fastEmit_AArch64ISD_TBL_rr(VT, RetVT, Op0, Op1);
14062 case AArch64ISD::TRN1: return fastEmit_AArch64ISD_TRN1_rr(VT, RetVT, Op0, Op1);
14063 case AArch64ISD::TRN2: return fastEmit_AArch64ISD_TRN2_rr(VT, RetVT, Op0, Op1);
14064 case AArch64ISD::UMULL: return fastEmit_AArch64ISD_UMULL_rr(VT, RetVT, Op0, Op1);
14065 case AArch64ISD::UQADD: return fastEmit_AArch64ISD_UQADD_rr(VT, RetVT, Op0, Op1);
14066 case AArch64ISD::UQRSHL: return fastEmit_AArch64ISD_UQRSHL_rr(VT, RetVT, Op0, Op1);
14067 case AArch64ISD::UQSHL: return fastEmit_AArch64ISD_UQSHL_rr(VT, RetVT, Op0, Op1);
14068 case AArch64ISD::UQSUB: return fastEmit_AArch64ISD_UQSUB_rr(VT, RetVT, Op0, Op1);
14069 case AArch64ISD::USQADD: return fastEmit_AArch64ISD_USQADD_rr(VT, RetVT, Op0, Op1);
14070 case AArch64ISD::UZP1: return fastEmit_AArch64ISD_UZP1_rr(VT, RetVT, Op0, Op1);
14071 case AArch64ISD::UZP2: return fastEmit_AArch64ISD_UZP2_rr(VT, RetVT, Op0, Op1);
14072 case AArch64ISD::ZIP1: return fastEmit_AArch64ISD_ZIP1_rr(VT, RetVT, Op0, Op1);
14073 case AArch64ISD::ZIP2: return fastEmit_AArch64ISD_ZIP2_rr(VT, RetVT, Op0, Op1);
14074 case ISD::ABDS: return fastEmit_ISD_ABDS_rr(VT, RetVT, Op0, Op1);
14075 case ISD::ABDU: return fastEmit_ISD_ABDU_rr(VT, RetVT, Op0, Op1);
14076 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
14077 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
14078 case ISD::AVGCEILS: return fastEmit_ISD_AVGCEILS_rr(VT, RetVT, Op0, Op1);
14079 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
14080 case ISD::AVGFLOORS: return fastEmit_ISD_AVGFLOORS_rr(VT, RetVT, Op0, Op1);
14081 case ISD::AVGFLOORU: return fastEmit_ISD_AVGFLOORU_rr(VT, RetVT, Op0, Op1);
14082 case ISD::CLMUL: return fastEmit_ISD_CLMUL_rr(VT, RetVT, Op0, Op1);
14083 case ISD::CONCAT_VECTORS: return fastEmit_ISD_CONCAT_VECTORS_rr(VT, RetVT, Op0, Op1);
14084 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
14085 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
14086 case ISD::FMAXIMUM: return fastEmit_ISD_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
14087 case ISD::FMAXNUM: return fastEmit_ISD_FMAXNUM_rr(VT, RetVT, Op0, Op1);
14088 case ISD::FMAXNUM_IEEE: return fastEmit_ISD_FMAXNUM_IEEE_rr(VT, RetVT, Op0, Op1);
14089 case ISD::FMINIMUM: return fastEmit_ISD_FMINIMUM_rr(VT, RetVT, Op0, Op1);
14090 case ISD::FMINNUM: return fastEmit_ISD_FMINNUM_rr(VT, RetVT, Op0, Op1);
14091 case ISD::FMINNUM_IEEE: return fastEmit_ISD_FMINNUM_IEEE_rr(VT, RetVT, Op0, Op1);
14092 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
14093 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
14094 case ISD::GET_ACTIVE_LANE_MASK: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_rr(VT, RetVT, Op0, Op1);
14095 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
14096 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
14097 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
14098 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
14099 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
14100 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
14101 case ISD::SDIV: return fastEmit_ISD_SDIV_rr(VT, RetVT, Op0, Op1);
14102 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
14103 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
14104 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
14105 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
14106 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
14107 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
14108 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1);
14109 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1);
14110 case ISD::STRICT_FMAXIMUM: return fastEmit_ISD_STRICT_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
14111 case ISD::STRICT_FMAXNUM: return fastEmit_ISD_STRICT_FMAXNUM_rr(VT, RetVT, Op0, Op1);
14112 case ISD::STRICT_FMINIMUM: return fastEmit_ISD_STRICT_FMINIMUM_rr(VT, RetVT, Op0, Op1);
14113 case ISD::STRICT_FMINNUM: return fastEmit_ISD_STRICT_FMINNUM_rr(VT, RetVT, Op0, Op1);
14114 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1);
14115 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1);
14116 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
14117 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
14118 case ISD::UDIV: return fastEmit_ISD_UDIV_rr(VT, RetVT, Op0, Op1);
14119 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
14120 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
14121 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
14122 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
14123 default: return Register();
14124 }
14125}
14126
14127// FastEmit functions for AArch64ISD::DUPLANE64.
14128
14129Register fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14130 if (RetVT.SimpleTy != MVT::v2i64)
14131 return Register();
14132 if ((Subtarget->isNeonAvailable())) {
14133 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i64lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14134 }
14135 return Register();
14136}
14137
14138Register fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14139 if (RetVT.SimpleTy != MVT::v2f64)
14140 return Register();
14141 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i64lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14142}
14143
14144Register fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14145 switch (VT.SimpleTy) {
14146 case MVT::v2i64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14147 case MVT::v2f64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14148 default: return Register();
14149 }
14150}
14151
14152// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14153
14154Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14155 if (RetVT.SimpleTy != MVT::i64)
14156 return Register();
14157 if ((Subtarget->isNeonAvailable())) {
14158 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi64, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14159 }
14160 return Register();
14161}
14162
14163Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14164 if (RetVT.SimpleTy != MVT::f64)
14165 return Register();
14166 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi64, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14167}
14168
14169Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14170 switch (VT.SimpleTy) {
14171 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14172 case MVT::v2f64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14173 default: return Register();
14174 }
14175}
14176
14177// Top-level FastEmit function.
14178
14179Register fastEmit_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14180 switch (Opcode) {
14181 case AArch64ISD::DUPLANE64: return fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1);
14182 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1);
14183 default: return Register();
14184 }
14185}
14186
14187// FastEmit functions for AArch64ISD::DUPLANE32.
14188
14189Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14190 if ((Subtarget->isNeonAvailable())) {
14191 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i32lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14192 }
14193 return Register();
14194}
14195
14196Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14197 if ((Subtarget->isNeonAvailable())) {
14198 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i32lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14199 }
14200 return Register();
14201}
14202
14203Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14204switch (RetVT.SimpleTy) {
14205 case MVT::v2i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Op0, imm1);
14206 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Op0, imm1);
14207 default: return Register();
14208}
14209}
14210
14211Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14212 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i32lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14213}
14214
14215Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14216 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i32lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14217}
14218
14219Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14220switch (RetVT.SimpleTy) {
14221 case MVT::v2f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Op0, imm1);
14222 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Op0, imm1);
14223 default: return Register();
14224}
14225}
14226
14227Register fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14228 switch (VT.SimpleTy) {
14229 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14230 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14231 default: return Register();
14232 }
14233}
14234
14235// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14236
14237Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14238 if (RetVT.SimpleTy != MVT::i32)
14239 return Register();
14240 if ((Subtarget->isNeonAvailable())) {
14241 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi32, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14242 }
14243 return Register();
14244}
14245
14246Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14247 if (RetVT.SimpleTy != MVT::f32)
14248 return Register();
14249 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi32, RC: &AArch64::FPR32RegClass, Op0, Imm: imm1);
14250}
14251
14252Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14253 switch (VT.SimpleTy) {
14254 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14255 case MVT::v4f32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14256 default: return Register();
14257 }
14258}
14259
14260// Top-level FastEmit function.
14261
14262Register fastEmit_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14263 switch (Opcode) {
14264 case AArch64ISD::DUPLANE32: return fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1);
14265 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1);
14266 default: return Register();
14267 }
14268}
14269
14270// FastEmit functions for AArch64ISD::DUPLANE16.
14271
14272Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14273 if ((Subtarget->isNeonAvailable())) {
14274 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14275 }
14276 return Register();
14277}
14278
14279Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14280 if ((Subtarget->isNeonAvailable())) {
14281 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14282 }
14283 return Register();
14284}
14285
14286Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14287switch (RetVT.SimpleTy) {
14288 case MVT::v4i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Op0, imm1);
14289 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Op0, imm1);
14290 default: return Register();
14291}
14292}
14293
14294Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14295 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14296}
14297
14298Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14299 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14300}
14301
14302Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14303switch (RetVT.SimpleTy) {
14304 case MVT::v4f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Op0, imm1);
14305 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Op0, imm1);
14306 default: return Register();
14307}
14308}
14309
14310Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14311 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14312}
14313
14314Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14315 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14316}
14317
14318Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14319switch (RetVT.SimpleTy) {
14320 case MVT::v4bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(Op0, imm1);
14321 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(Op0, imm1);
14322 default: return Register();
14323}
14324}
14325
14326Register fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14327 switch (VT.SimpleTy) {
14328 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14329 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14330 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14331 default: return Register();
14332 }
14333}
14334
14335// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14336
14337Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14338 if (RetVT.SimpleTy != MVT::i32)
14339 return Register();
14340 if ((Subtarget->isNeonAvailable())) {
14341 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi16, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14342 }
14343 return Register();
14344}
14345
14346Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14347 if (RetVT.SimpleTy != MVT::f16)
14348 return Register();
14349 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi16, RC: &AArch64::FPR16RegClass, Op0, Imm: imm1);
14350}
14351
14352Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14353 if (RetVT.SimpleTy != MVT::bf16)
14354 return Register();
14355 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi16, RC: &AArch64::FPR16RegClass, Op0, Imm: imm1);
14356}
14357
14358Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14359 switch (VT.SimpleTy) {
14360 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14361 case MVT::v8f16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14362 case MVT::v8bf16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14363 default: return Register();
14364 }
14365}
14366
14367// Top-level FastEmit function.
14368
14369Register fastEmit_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14370 switch (Opcode) {
14371 case AArch64ISD::DUPLANE16: return fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1);
14372 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1);
14373 default: return Register();
14374 }
14375}
14376
14377// FastEmit functions for AArch64ISD::DUPLANE8.
14378
14379Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Register Op0, uint64_t imm1) {
14380 if ((Subtarget->isNeonAvailable())) {
14381 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i8lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14382 }
14383 return Register();
14384}
14385
14386Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Register Op0, uint64_t imm1) {
14387 if ((Subtarget->isNeonAvailable())) {
14388 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv16i8lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14389 }
14390 return Register();
14391}
14392
14393Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, Register Op0, uint64_t imm1) {
14394switch (RetVT.SimpleTy) {
14395 case MVT::v8i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Op0, imm1);
14396 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Op0, imm1);
14397 default: return Register();
14398}
14399}
14400
14401Register fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14402 switch (VT.SimpleTy) {
14403 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1);
14404 default: return Register();
14405 }
14406}
14407
14408// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14409
14410Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, Register Op0, uint64_t imm1) {
14411 if (RetVT.SimpleTy != MVT::i32)
14412 return Register();
14413 if ((Subtarget->isNeonAvailable())) {
14414 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi8, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14415 }
14416 return Register();
14417}
14418
14419Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14420 switch (VT.SimpleTy) {
14421 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1);
14422 default: return Register();
14423 }
14424}
14425
14426// Top-level FastEmit function.
14427
14428Register fastEmit_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14429 switch (Opcode) {
14430 case AArch64ISD::DUPLANE8: return fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1);
14431 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1);
14432 default: return Register();
14433 }
14434}
14435
14436// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14437
14438Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14439 if (RetVT.SimpleTy != MVT::i32)
14440 return Register();
14441 if ((Subtarget->hasNEON())) {
14442 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi8_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14443 }
14444 return Register();
14445}
14446
14447Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14448 if (RetVT.SimpleTy != MVT::i32)
14449 return Register();
14450 if ((Subtarget->hasNEON())) {
14451 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi16_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14452 }
14453 return Register();
14454}
14455
14456Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14457 if (RetVT.SimpleTy != MVT::i32)
14458 return Register();
14459 if ((Subtarget->hasNEON())) {
14460 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi32_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14461 }
14462 return Register();
14463}
14464
14465Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14466 if (RetVT.SimpleTy != MVT::i64)
14467 return Register();
14468 if ((Subtarget->hasNEON())) {
14469 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi64_idx0, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14470 }
14471 return Register();
14472}
14473
14474Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14475 switch (VT.SimpleTy) {
14476 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14477 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14478 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14479 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14480 default: return Register();
14481 }
14482}
14483
14484// Top-level FastEmit function.
14485
14486Register fastEmit_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14487 switch (Opcode) {
14488 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(VT, RetVT, Op0, imm1);
14489 default: return Register();
14490 }
14491}
14492
14493// FastEmit functions for ISD::SMAX.
14494
14495Register fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14496 if (RetVT.SimpleTy != MVT::i32)
14497 return Register();
14498 if ((Subtarget->hasCSSC())) {
14499 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMAXWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14500 }
14501 return Register();
14502}
14503
14504Register fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14505 switch (VT.SimpleTy) {
14506 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1);
14507 default: return Register();
14508 }
14509}
14510
14511// FastEmit functions for ISD::SMIN.
14512
14513Register fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14514 if (RetVT.SimpleTy != MVT::i32)
14515 return Register();
14516 if ((Subtarget->hasCSSC())) {
14517 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMINWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14518 }
14519 return Register();
14520}
14521
14522Register fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14523 switch (VT.SimpleTy) {
14524 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1);
14525 default: return Register();
14526 }
14527}
14528
14529// Top-level FastEmit function.
14530
14531Register fastEmit_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14532 switch (Opcode) {
14533 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1);
14534 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1);
14535 default: return Register();
14536 }
14537}
14538
14539// FastEmit functions for ISD::SMAX.
14540
14541Register fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14542 if (RetVT.SimpleTy != MVT::i64)
14543 return Register();
14544 if ((Subtarget->hasCSSC())) {
14545 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMAXXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14546 }
14547 return Register();
14548}
14549
14550Register fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14551 switch (VT.SimpleTy) {
14552 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1);
14553 default: return Register();
14554 }
14555}
14556
14557// FastEmit functions for ISD::SMIN.
14558
14559Register fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14560 if (RetVT.SimpleTy != MVT::i64)
14561 return Register();
14562 if ((Subtarget->hasCSSC())) {
14563 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMINXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14564 }
14565 return Register();
14566}
14567
14568Register fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14569 switch (VT.SimpleTy) {
14570 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1);
14571 default: return Register();
14572 }
14573}
14574
14575// Top-level FastEmit function.
14576
14577Register fastEmit_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14578 switch (Opcode) {
14579 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1);
14580 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1);
14581 default: return Register();
14582 }
14583}
14584
14585// FastEmit functions for ISD::UMAX.
14586
14587Register fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14588 if (RetVT.SimpleTy != MVT::i32)
14589 return Register();
14590 if ((Subtarget->hasCSSC())) {
14591 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMAXWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14592 }
14593 return Register();
14594}
14595
14596Register fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14597 switch (VT.SimpleTy) {
14598 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1);
14599 default: return Register();
14600 }
14601}
14602
14603// FastEmit functions for ISD::UMIN.
14604
14605Register fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14606 if (RetVT.SimpleTy != MVT::i32)
14607 return Register();
14608 if ((Subtarget->hasCSSC())) {
14609 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMINWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14610 }
14611 return Register();
14612}
14613
14614Register fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14615 switch (VT.SimpleTy) {
14616 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1);
14617 default: return Register();
14618 }
14619}
14620
14621// Top-level FastEmit function.
14622
14623Register fastEmit_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14624 switch (Opcode) {
14625 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1);
14626 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1);
14627 default: return Register();
14628 }
14629}
14630
14631// FastEmit functions for ISD::UMAX.
14632
14633Register fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14634 if (RetVT.SimpleTy != MVT::i64)
14635 return Register();
14636 if ((Subtarget->hasCSSC())) {
14637 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMAXXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14638 }
14639 return Register();
14640}
14641
14642Register fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14643 switch (VT.SimpleTy) {
14644 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1);
14645 default: return Register();
14646 }
14647}
14648
14649// FastEmit functions for ISD::UMIN.
14650
14651Register fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14652 if (RetVT.SimpleTy != MVT::i64)
14653 return Register();
14654 if ((Subtarget->hasCSSC())) {
14655 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMINXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14656 }
14657 return Register();
14658}
14659
14660Register fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14661 switch (VT.SimpleTy) {
14662 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1);
14663 default: return Register();
14664 }
14665}
14666
14667// Top-level FastEmit function.
14668
14669Register fastEmit_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14670 switch (Opcode) {
14671 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1);
14672 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1);
14673 default: return Register();
14674 }
14675}
14676
14677// FastEmit functions for ISD::Constant.
14678
14679Register fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
14680 if (RetVT.SimpleTy != MVT::i32)
14681 return Register();
14682 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVi32imm, RC: &AArch64::GPR32RegClass, Imm: imm0);
14683}
14684
14685Register fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) {
14686 if (RetVT.SimpleTy != MVT::i64)
14687 return Register();
14688 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVi64imm, RC: &AArch64::GPR64RegClass, Imm: imm0);
14689}
14690
14691Register fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
14692 switch (VT.SimpleTy) {
14693 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
14694 case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0);
14695 default: return Register();
14696 }
14697}
14698
14699// Top-level FastEmit function.
14700
14701Register fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
14702 if (VT == MVT::i32 && Predicate_imm0_255(Imm: imm0))
14703 if (Register Reg = fastEmit_i_Predicate_imm0_255(VT, RetVT, Opcode, imm0))
14704 return Reg;
14705
14706 if (VT == MVT::i32 && Predicate_simm6_32b(Imm: imm0))
14707 if (Register Reg = fastEmit_i_Predicate_simm6_32b(VT, RetVT, Opcode, imm0))
14708 return Reg;
14709
14710 switch (Opcode) {
14711 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
14712 default: return Register();
14713 }
14714}
14715
14716// FastEmit functions for AArch64ISD::FMOV.
14717
14718Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(uint64_t imm0) {
14719 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
14720 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv4f16_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
14721 }
14722 return Register();
14723}
14724
14725Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(uint64_t imm0) {
14726 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
14727 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv8f16_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14728 }
14729 return Register();
14730}
14731
14732Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(uint64_t imm0) {
14733 if ((Subtarget->isNeonAvailable())) {
14734 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv2f32_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
14735 }
14736 return Register();
14737}
14738
14739Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(uint64_t imm0) {
14740 if ((Subtarget->isNeonAvailable())) {
14741 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv4f32_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14742 }
14743 return Register();
14744}
14745
14746Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(uint64_t imm0) {
14747 if ((Subtarget->isNeonAvailable())) {
14748 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv2f64_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14749 }
14750 return Register();
14751}
14752
14753Register fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
14754switch (RetVT.SimpleTy) {
14755 case MVT::v4f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(imm0);
14756 case MVT::v8f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(imm0);
14757 case MVT::v2f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(imm0);
14758 case MVT::v4f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(imm0);
14759 case MVT::v2f64: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(imm0);
14760 default: return Register();
14761}
14762}
14763
14764Register fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
14765 switch (VT.SimpleTy) {
14766 case MVT::i32: return fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
14767 default: return Register();
14768 }
14769}
14770
14771// FastEmit functions for AArch64ISD::MOVI.
14772
14773Register fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(uint64_t imm0) {
14774 if ((Subtarget->isNeonAvailable())) {
14775 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv8b_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
14776 }
14777 return Register();
14778}
14779
14780Register fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(uint64_t imm0) {
14781 if ((Subtarget->isNeonAvailable())) {
14782 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv16b_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14783 }
14784 return Register();
14785}
14786
14787Register fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
14788switch (RetVT.SimpleTy) {
14789 case MVT::v8i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(imm0);
14790 case MVT::v16i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(imm0);
14791 default: return Register();
14792}
14793}
14794
14795Register fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
14796 switch (VT.SimpleTy) {
14797 case MVT::i32: return fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
14798 default: return Register();
14799 }
14800}
14801
14802// FastEmit functions for AArch64ISD::MOVIedit.
14803
14804Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(uint64_t imm0) {
14805 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVID, RC: &AArch64::FPR64RegClass, Imm: imm0);
14806}
14807
14808Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(uint64_t imm0) {
14809 if ((Subtarget->isNeonAvailable())) {
14810 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv2d_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14811 }
14812 return Register();
14813}
14814
14815Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
14816switch (RetVT.SimpleTy) {
14817 case MVT::f64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(imm0);
14818 case MVT::v2i64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(imm0);
14819 default: return Register();
14820}
14821}
14822
14823Register fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
14824 switch (VT.SimpleTy) {
14825 case MVT::i32: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
14826 default: return Register();
14827 }
14828}
14829
14830// Top-level FastEmit function.
14831
14832Register fastEmit_i_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
14833 switch (Opcode) {
14834 case AArch64ISD::FMOV: return fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(VT, RetVT, imm0);
14835 case AArch64ISD::MOVI: return fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(VT, RetVT, imm0);
14836 case AArch64ISD::MOVIedit: return fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(VT, RetVT, imm0);
14837 default: return Register();
14838 }
14839}
14840
14841// FastEmit functions for AArch64ISD::RDSVL.
14842
14843Register fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(MVT RetVT, uint64_t imm0) {
14844 if (RetVT.SimpleTy != MVT::i64)
14845 return Register();
14846 if ((Subtarget->hasSME())) {
14847 return fastEmitInst_i(MachineInstOpcode: AArch64::RDSVLI_XI, RC: &AArch64::GPR64RegClass, Imm: imm0);
14848 }
14849 return Register();
14850}
14851
14852Register fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(MVT VT, MVT RetVT, uint64_t imm0) {
14853 switch (VT.SimpleTy) {
14854 case MVT::i32: return fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(RetVT, imm0);
14855 default: return Register();
14856 }
14857}
14858
14859// Top-level FastEmit function.
14860
14861Register fastEmit_i_Predicate_simm6_32b(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
14862 switch (Opcode) {
14863 case AArch64ISD::RDSVL: return fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(VT, RetVT, imm0);
14864 default: return Register();
14865 }
14866}
14867
14868