1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the AArch64 target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_imm0_31(int64_t Imm) {
12
13 return ((uint64_t)Imm) < 32;
14
15}
16static bool Predicate_imm0_63(int64_t Imm) {
17
18 return ((uint64_t)Imm) < 64;
19
20}
21static bool Predicate_VectorIndexD(int64_t Imm) {
22 return ((uint64_t)Imm) < 2;
23}
24static bool Predicate_VectorIndexS(int64_t Imm) {
25 return ((uint64_t)Imm) < 4;
26}
27static bool Predicate_VectorIndexH(int64_t Imm) {
28 return ((uint64_t)Imm) < 8;
29}
30static bool Predicate_VectorIndexB(int64_t Imm) {
31 return ((uint64_t)Imm) < 16;
32}
33static bool Predicate_VectorIndex0(int64_t Imm) {
34 return ((uint64_t)Imm) == 0;
35}
36static bool Predicate_imm0_255(int64_t Imm) {
37
38 return ((uint32_t)Imm) < 256;
39
40}
41static bool Predicate_simm8_32b(int64_t Imm) {
42 return Imm >= -128 && Imm < 128;
43}
44static bool Predicate_simm8_64b(int64_t Imm) {
45 return Imm >= -128 && Imm < 128;
46}
47static bool Predicate_uimm8_32b(int64_t Imm) {
48 return Imm >= 0 && Imm < 256;
49}
50static bool Predicate_uimm8_64b(int64_t Imm) {
51 return Imm >= 0 && Imm < 256;
52}
53static bool Predicate_simm6_32b(int64_t Imm) {
54 return Imm >= -32 && Imm < 32;
55}
56
57
58// FastEmit functions for AArch64ISD::ENTRY_PSTATE_SM.
59
60Register fastEmit_AArch64ISD_ENTRY_PSTATE_SM_MVT_i64_(MVT RetVT) {
61 if (RetVT.SimpleTy != MVT::i64)
62 return Register();
63 return fastEmitInst_(MachineInstOpcode: AArch64::EntryPStateSM, RC: &AArch64::GPR64RegClass);
64}
65
66Register fastEmit_AArch64ISD_ENTRY_PSTATE_SM_(MVT VT, MVT RetVT) {
67 switch (VT.SimpleTy) {
68 case MVT::i64: return fastEmit_AArch64ISD_ENTRY_PSTATE_SM_MVT_i64_(RetVT);
69 default: return Register();
70 }
71}
72
73// FastEmit functions for AArch64ISD::THREAD_POINTER.
74
75Register fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(MVT RetVT) {
76 if (RetVT.SimpleTy != MVT::i64)
77 return Register();
78 return fastEmitInst_(MachineInstOpcode: AArch64::MOVbaseTLS, RC: &AArch64::GPR64RegClass);
79}
80
81Register fastEmit_AArch64ISD_THREAD_POINTER_(MVT VT, MVT RetVT) {
82 switch (VT.SimpleTy) {
83 case MVT::i64: return fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(RetVT);
84 default: return Register();
85 }
86}
87
88// Top-level FastEmit function.
89
90Register fastEmit_(MVT VT, MVT RetVT, unsigned Opcode) override {
91 switch (Opcode) {
92 case AArch64ISD::ENTRY_PSTATE_SM: return fastEmit_AArch64ISD_ENTRY_PSTATE_SM_(VT, RetVT);
93 case AArch64ISD::THREAD_POINTER: return fastEmit_AArch64ISD_THREAD_POINTER_(VT, RetVT);
94 default: return Register();
95 }
96}
97
98// FastEmit functions for AArch64ISD::COALESCER_BARRIER.
99
100Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_bf16_r(MVT RetVT, Register Op0) {
101 if (RetVT.SimpleTy != MVT::bf16)
102 return Register();
103 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR16, RC: &AArch64::FPR16RegClass, Op0);
104}
105
106Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f16_r(MVT RetVT, Register Op0) {
107 if (RetVT.SimpleTy != MVT::f16)
108 return Register();
109 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR16, RC: &AArch64::FPR16RegClass, Op0);
110}
111
112Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f32_r(MVT RetVT, Register Op0) {
113 if (RetVT.SimpleTy != MVT::f32)
114 return Register();
115 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR32, RC: &AArch64::FPR32RegClass, Op0);
116}
117
118Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f64_r(MVT RetVT, Register Op0) {
119 if (RetVT.SimpleTy != MVT::f64)
120 return Register();
121 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
122}
123
124Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f128_r(MVT RetVT, Register Op0) {
125 if (RetVT.SimpleTy != MVT::f128)
126 return Register();
127 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
128}
129
130Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i8_r(MVT RetVT, Register Op0) {
131 if (RetVT.SimpleTy != MVT::v8i8)
132 return Register();
133 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
134}
135
136Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v16i8_r(MVT RetVT, Register Op0) {
137 if (RetVT.SimpleTy != MVT::v16i8)
138 return Register();
139 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
140}
141
142Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i16_r(MVT RetVT, Register Op0) {
143 if (RetVT.SimpleTy != MVT::v4i16)
144 return Register();
145 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
146}
147
148Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i16_r(MVT RetVT, Register Op0) {
149 if (RetVT.SimpleTy != MVT::v8i16)
150 return Register();
151 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
152}
153
154Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i32_r(MVT RetVT, Register Op0) {
155 if (RetVT.SimpleTy != MVT::v2i32)
156 return Register();
157 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
158}
159
160Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i32_r(MVT RetVT, Register Op0) {
161 if (RetVT.SimpleTy != MVT::v4i32)
162 return Register();
163 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
164}
165
166Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1i64_r(MVT RetVT, Register Op0) {
167 if (RetVT.SimpleTy != MVT::v1i64)
168 return Register();
169 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
170}
171
172Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i64_r(MVT RetVT, Register Op0) {
173 if (RetVT.SimpleTy != MVT::v2i64)
174 return Register();
175 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
176}
177
178Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f16_r(MVT RetVT, Register Op0) {
179 if (RetVT.SimpleTy != MVT::v4f16)
180 return Register();
181 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
182}
183
184Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8f16_r(MVT RetVT, Register Op0) {
185 if (RetVT.SimpleTy != MVT::v8f16)
186 return Register();
187 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
188}
189
190Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4bf16_r(MVT RetVT, Register Op0) {
191 if (RetVT.SimpleTy != MVT::v4bf16)
192 return Register();
193 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
194}
195
196Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8bf16_r(MVT RetVT, Register Op0) {
197 if (RetVT.SimpleTy != MVT::v8bf16)
198 return Register();
199 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
200}
201
202Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f32_r(MVT RetVT, Register Op0) {
203 if (RetVT.SimpleTy != MVT::v2f32)
204 return Register();
205 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
206}
207
208Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f32_r(MVT RetVT, Register Op0) {
209 if (RetVT.SimpleTy != MVT::v4f32)
210 return Register();
211 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
212}
213
214Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1f64_r(MVT RetVT, Register Op0) {
215 if (RetVT.SimpleTy != MVT::v1f64)
216 return Register();
217 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
218}
219
220Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f64_r(MVT RetVT, Register Op0) {
221 if (RetVT.SimpleTy != MVT::v2f64)
222 return Register();
223 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
224}
225
226Register fastEmit_AArch64ISD_COALESCER_BARRIER_r(MVT VT, MVT RetVT, Register Op0) {
227 switch (VT.SimpleTy) {
228 case MVT::bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_bf16_r(RetVT, Op0);
229 case MVT::f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f16_r(RetVT, Op0);
230 case MVT::f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f32_r(RetVT, Op0);
231 case MVT::f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f64_r(RetVT, Op0);
232 case MVT::f128: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f128_r(RetVT, Op0);
233 case MVT::v8i8: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i8_r(RetVT, Op0);
234 case MVT::v16i8: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v16i8_r(RetVT, Op0);
235 case MVT::v4i16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i16_r(RetVT, Op0);
236 case MVT::v8i16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i16_r(RetVT, Op0);
237 case MVT::v2i32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i32_r(RetVT, Op0);
238 case MVT::v4i32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i32_r(RetVT, Op0);
239 case MVT::v1i64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1i64_r(RetVT, Op0);
240 case MVT::v2i64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i64_r(RetVT, Op0);
241 case MVT::v4f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f16_r(RetVT, Op0);
242 case MVT::v8f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8f16_r(RetVT, Op0);
243 case MVT::v4bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4bf16_r(RetVT, Op0);
244 case MVT::v8bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8bf16_r(RetVT, Op0);
245 case MVT::v2f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f32_r(RetVT, Op0);
246 case MVT::v4f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f32_r(RetVT, Op0);
247 case MVT::v1f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1f64_r(RetVT, Op0);
248 case MVT::v2f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f64_r(RetVT, Op0);
249 default: return Register();
250 }
251}
252
253// FastEmit functions for AArch64ISD::DUP.
254
255Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Register Op0) {
256 if ((Subtarget->isNeonAvailable())) {
257 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv8i8gpr, RC: &AArch64::FPR64RegClass, Op0);
258 }
259 return Register();
260}
261
262Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Register Op0) {
263 if ((Subtarget->isNeonAvailable())) {
264 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv16i8gpr, RC: &AArch64::FPR128RegClass, Op0);
265 }
266 return Register();
267}
268
269Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Register Op0) {
270 if ((Subtarget->isNeonAvailable())) {
271 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv4i16gpr, RC: &AArch64::FPR64RegClass, Op0);
272 }
273 return Register();
274}
275
276Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Register Op0) {
277 if ((Subtarget->isNeonAvailable())) {
278 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv8i16gpr, RC: &AArch64::FPR128RegClass, Op0);
279 }
280 return Register();
281}
282
283Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Register Op0) {
284 if ((Subtarget->isNeonAvailable())) {
285 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv2i32gpr, RC: &AArch64::FPR64RegClass, Op0);
286 }
287 return Register();
288}
289
290Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Register Op0) {
291 if ((Subtarget->isNeonAvailable())) {
292 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv4i32gpr, RC: &AArch64::FPR128RegClass, Op0);
293 }
294 return Register();
295}
296
297Register fastEmit_AArch64ISD_DUP_MVT_i32_r(MVT RetVT, Register Op0) {
298switch (RetVT.SimpleTy) {
299 case MVT::v8i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Op0);
300 case MVT::v16i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Op0);
301 case MVT::v4i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Op0);
302 case MVT::v8i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Op0);
303 case MVT::v2i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Op0);
304 case MVT::v4i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Op0);
305 default: return Register();
306}
307}
308
309Register fastEmit_AArch64ISD_DUP_MVT_i64_r(MVT RetVT, Register Op0) {
310 if (RetVT.SimpleTy != MVT::v2i64)
311 return Register();
312 if ((Subtarget->isNeonAvailable())) {
313 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv2i64gpr, RC: &AArch64::FPR128RegClass, Op0);
314 }
315 return Register();
316}
317
318Register fastEmit_AArch64ISD_DUP_r(MVT VT, MVT RetVT, Register Op0) {
319 switch (VT.SimpleTy) {
320 case MVT::i32: return fastEmit_AArch64ISD_DUP_MVT_i32_r(RetVT, Op0);
321 case MVT::i64: return fastEmit_AArch64ISD_DUP_MVT_i64_r(RetVT, Op0);
322 default: return Register();
323 }
324}
325
326// FastEmit functions for AArch64ISD::FCVTXN.
327
328Register fastEmit_AArch64ISD_FCVTXN_MVT_f64_r(MVT RetVT, Register Op0) {
329 if (RetVT.SimpleTy != MVT::f32)
330 return Register();
331 if ((Subtarget->isNeonAvailable())) {
332 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTXNv1i64, RC: &AArch64::FPR32RegClass, Op0);
333 }
334 return Register();
335}
336
337Register fastEmit_AArch64ISD_FCVTXN_MVT_v2f64_r(MVT RetVT, Register Op0) {
338 if (RetVT.SimpleTy != MVT::v2f32)
339 return Register();
340 if ((Subtarget->isNeonAvailable())) {
341 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTXNv2f32, RC: &AArch64::FPR64RegClass, Op0);
342 }
343 return Register();
344}
345
346Register fastEmit_AArch64ISD_FCVTXN_r(MVT VT, MVT RetVT, Register Op0) {
347 switch (VT.SimpleTy) {
348 case MVT::f64: return fastEmit_AArch64ISD_FCVTXN_MVT_f64_r(RetVT, Op0);
349 case MVT::v2f64: return fastEmit_AArch64ISD_FCVTXN_MVT_v2f64_r(RetVT, Op0);
350 default: return Register();
351 }
352}
353
354// FastEmit functions for AArch64ISD::FRECPE.
355
356Register fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(MVT RetVT, Register Op0) {
357 if (RetVT.SimpleTy != MVT::v2f32)
358 return Register();
359 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv2f32, RC: &AArch64::FPR64RegClass, Op0);
360}
361
362Register fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(MVT RetVT, Register Op0) {
363 if (RetVT.SimpleTy != MVT::v4f32)
364 return Register();
365 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv4f32, RC: &AArch64::FPR128RegClass, Op0);
366}
367
368Register fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(MVT RetVT, Register Op0) {
369 if (RetVT.SimpleTy != MVT::v2f64)
370 return Register();
371 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv2f64, RC: &AArch64::FPR128RegClass, Op0);
372}
373
374Register fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
375 if (RetVT.SimpleTy != MVT::nxv8f16)
376 return Register();
377 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
378 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
379 }
380 return Register();
381}
382
383Register fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
384 if (RetVT.SimpleTy != MVT::nxv4f32)
385 return Register();
386 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
387 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
388 }
389 return Register();
390}
391
392Register fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
393 if (RetVT.SimpleTy != MVT::nxv2f64)
394 return Register();
395 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
396 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
397 }
398 return Register();
399}
400
401Register fastEmit_AArch64ISD_FRECPE_r(MVT VT, MVT RetVT, Register Op0) {
402 switch (VT.SimpleTy) {
403 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(RetVT, Op0);
404 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(RetVT, Op0);
405 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(RetVT, Op0);
406 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(RetVT, Op0);
407 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(RetVT, Op0);
408 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(RetVT, Op0);
409 default: return Register();
410 }
411}
412
413// FastEmit functions for AArch64ISD::FRSQRTE.
414
415Register fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(MVT RetVT, Register Op0) {
416 if (RetVT.SimpleTy != MVT::v2f32)
417 return Register();
418 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv2f32, RC: &AArch64::FPR64RegClass, Op0);
419}
420
421Register fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(MVT RetVT, Register Op0) {
422 if (RetVT.SimpleTy != MVT::v4f32)
423 return Register();
424 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv4f32, RC: &AArch64::FPR128RegClass, Op0);
425}
426
427Register fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(MVT RetVT, Register Op0) {
428 if (RetVT.SimpleTy != MVT::v2f64)
429 return Register();
430 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv2f64, RC: &AArch64::FPR128RegClass, Op0);
431}
432
433Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
434 if (RetVT.SimpleTy != MVT::nxv8f16)
435 return Register();
436 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
437 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
438 }
439 return Register();
440}
441
442Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
443 if (RetVT.SimpleTy != MVT::nxv4f32)
444 return Register();
445 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
446 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
447 }
448 return Register();
449}
450
451Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
452 if (RetVT.SimpleTy != MVT::nxv2f64)
453 return Register();
454 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
455 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
456 }
457 return Register();
458}
459
460Register fastEmit_AArch64ISD_FRSQRTE_r(MVT VT, MVT RetVT, Register Op0) {
461 switch (VT.SimpleTy) {
462 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(RetVT, Op0);
463 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(RetVT, Op0);
464 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(RetVT, Op0);
465 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(RetVT, Op0);
466 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(RetVT, Op0);
467 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(RetVT, Op0);
468 default: return Register();
469 }
470}
471
472// FastEmit functions for AArch64ISD::REV16.
473
474Register fastEmit_AArch64ISD_REV16_MVT_i32_r(MVT RetVT, Register Op0) {
475 if (RetVT.SimpleTy != MVT::i32)
476 return Register();
477 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16Wr, RC: &AArch64::GPR32RegClass, Op0);
478}
479
480Register fastEmit_AArch64ISD_REV16_MVT_i64_r(MVT RetVT, Register Op0) {
481 if (RetVT.SimpleTy != MVT::i64)
482 return Register();
483 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16Xr, RC: &AArch64::GPR64RegClass, Op0);
484}
485
486Register fastEmit_AArch64ISD_REV16_r(MVT VT, MVT RetVT, Register Op0) {
487 switch (VT.SimpleTy) {
488 case MVT::i32: return fastEmit_AArch64ISD_REV16_MVT_i32_r(RetVT, Op0);
489 case MVT::i64: return fastEmit_AArch64ISD_REV16_MVT_i64_r(RetVT, Op0);
490 default: return Register();
491 }
492}
493
494// FastEmit functions for AArch64ISD::REV32.
495
496Register fastEmit_AArch64ISD_REV32_MVT_v8i8_r(MVT RetVT, Register Op0) {
497 if (RetVT.SimpleTy != MVT::v8i8)
498 return Register();
499 if ((Subtarget->isNeonAvailable())) {
500 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
501 }
502 return Register();
503}
504
505Register fastEmit_AArch64ISD_REV32_MVT_v16i8_r(MVT RetVT, Register Op0) {
506 if (RetVT.SimpleTy != MVT::v16i8)
507 return Register();
508 if ((Subtarget->isNeonAvailable())) {
509 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
510 }
511 return Register();
512}
513
514Register fastEmit_AArch64ISD_REV32_MVT_v4i16_r(MVT RetVT, Register Op0) {
515 if (RetVT.SimpleTy != MVT::v4i16)
516 return Register();
517 if ((Subtarget->isNeonAvailable())) {
518 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
519 }
520 return Register();
521}
522
523Register fastEmit_AArch64ISD_REV32_MVT_v8i16_r(MVT RetVT, Register Op0) {
524 if (RetVT.SimpleTy != MVT::v8i16)
525 return Register();
526 if ((Subtarget->isNeonAvailable())) {
527 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
528 }
529 return Register();
530}
531
532Register fastEmit_AArch64ISD_REV32_MVT_v4f16_r(MVT RetVT, Register Op0) {
533 if (RetVT.SimpleTy != MVT::v4f16)
534 return Register();
535 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
536}
537
538Register fastEmit_AArch64ISD_REV32_MVT_v8f16_r(MVT RetVT, Register Op0) {
539 if (RetVT.SimpleTy != MVT::v8f16)
540 return Register();
541 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
542}
543
544Register fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(MVT RetVT, Register Op0) {
545 if (RetVT.SimpleTy != MVT::v4bf16)
546 return Register();
547 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
548}
549
550Register fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(MVT RetVT, Register Op0) {
551 if (RetVT.SimpleTy != MVT::v8bf16)
552 return Register();
553 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
554}
555
556Register fastEmit_AArch64ISD_REV32_r(MVT VT, MVT RetVT, Register Op0) {
557 switch (VT.SimpleTy) {
558 case MVT::v8i8: return fastEmit_AArch64ISD_REV32_MVT_v8i8_r(RetVT, Op0);
559 case MVT::v16i8: return fastEmit_AArch64ISD_REV32_MVT_v16i8_r(RetVT, Op0);
560 case MVT::v4i16: return fastEmit_AArch64ISD_REV32_MVT_v4i16_r(RetVT, Op0);
561 case MVT::v8i16: return fastEmit_AArch64ISD_REV32_MVT_v8i16_r(RetVT, Op0);
562 case MVT::v4f16: return fastEmit_AArch64ISD_REV32_MVT_v4f16_r(RetVT, Op0);
563 case MVT::v8f16: return fastEmit_AArch64ISD_REV32_MVT_v8f16_r(RetVT, Op0);
564 case MVT::v4bf16: return fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(RetVT, Op0);
565 case MVT::v8bf16: return fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(RetVT, Op0);
566 default: return Register();
567 }
568}
569
570// FastEmit functions for AArch64ISD::REV64.
571
572Register fastEmit_AArch64ISD_REV64_MVT_v8i8_r(MVT RetVT, Register Op0) {
573 if (RetVT.SimpleTy != MVT::v8i8)
574 return Register();
575 if ((Subtarget->isNeonAvailable())) {
576 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
577 }
578 return Register();
579}
580
581Register fastEmit_AArch64ISD_REV64_MVT_v16i8_r(MVT RetVT, Register Op0) {
582 if (RetVT.SimpleTy != MVT::v16i8)
583 return Register();
584 if ((Subtarget->isNeonAvailable())) {
585 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
586 }
587 return Register();
588}
589
590Register fastEmit_AArch64ISD_REV64_MVT_v4i16_r(MVT RetVT, Register Op0) {
591 if (RetVT.SimpleTy != MVT::v4i16)
592 return Register();
593 if ((Subtarget->isNeonAvailable())) {
594 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
595 }
596 return Register();
597}
598
599Register fastEmit_AArch64ISD_REV64_MVT_v8i16_r(MVT RetVT, Register Op0) {
600 if (RetVT.SimpleTy != MVT::v8i16)
601 return Register();
602 if ((Subtarget->isNeonAvailable())) {
603 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
604 }
605 return Register();
606}
607
608Register fastEmit_AArch64ISD_REV64_MVT_v2i32_r(MVT RetVT, Register Op0) {
609 if (RetVT.SimpleTy != MVT::v2i32)
610 return Register();
611 if ((Subtarget->isNeonAvailable())) {
612 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
613 }
614 return Register();
615}
616
617Register fastEmit_AArch64ISD_REV64_MVT_v4i32_r(MVT RetVT, Register Op0) {
618 if (RetVT.SimpleTy != MVT::v4i32)
619 return Register();
620 if ((Subtarget->isNeonAvailable())) {
621 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
622 }
623 return Register();
624}
625
626Register fastEmit_AArch64ISD_REV64_MVT_v4f16_r(MVT RetVT, Register Op0) {
627 if (RetVT.SimpleTy != MVT::v4f16)
628 return Register();
629 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
630}
631
632Register fastEmit_AArch64ISD_REV64_MVT_v8f16_r(MVT RetVT, Register Op0) {
633 if (RetVT.SimpleTy != MVT::v8f16)
634 return Register();
635 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
636}
637
638Register fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(MVT RetVT, Register Op0) {
639 if (RetVT.SimpleTy != MVT::v4bf16)
640 return Register();
641 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
642}
643
644Register fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(MVT RetVT, Register Op0) {
645 if (RetVT.SimpleTy != MVT::v8bf16)
646 return Register();
647 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
648}
649
650Register fastEmit_AArch64ISD_REV64_MVT_v2f32_r(MVT RetVT, Register Op0) {
651 if (RetVT.SimpleTy != MVT::v2f32)
652 return Register();
653 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
654}
655
656Register fastEmit_AArch64ISD_REV64_MVT_v4f32_r(MVT RetVT, Register Op0) {
657 if (RetVT.SimpleTy != MVT::v4f32)
658 return Register();
659 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
660}
661
662Register fastEmit_AArch64ISD_REV64_r(MVT VT, MVT RetVT, Register Op0) {
663 switch (VT.SimpleTy) {
664 case MVT::v8i8: return fastEmit_AArch64ISD_REV64_MVT_v8i8_r(RetVT, Op0);
665 case MVT::v16i8: return fastEmit_AArch64ISD_REV64_MVT_v16i8_r(RetVT, Op0);
666 case MVT::v4i16: return fastEmit_AArch64ISD_REV64_MVT_v4i16_r(RetVT, Op0);
667 case MVT::v8i16: return fastEmit_AArch64ISD_REV64_MVT_v8i16_r(RetVT, Op0);
668 case MVT::v2i32: return fastEmit_AArch64ISD_REV64_MVT_v2i32_r(RetVT, Op0);
669 case MVT::v4i32: return fastEmit_AArch64ISD_REV64_MVT_v4i32_r(RetVT, Op0);
670 case MVT::v4f16: return fastEmit_AArch64ISD_REV64_MVT_v4f16_r(RetVT, Op0);
671 case MVT::v8f16: return fastEmit_AArch64ISD_REV64_MVT_v8f16_r(RetVT, Op0);
672 case MVT::v4bf16: return fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(RetVT, Op0);
673 case MVT::v8bf16: return fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(RetVT, Op0);
674 case MVT::v2f32: return fastEmit_AArch64ISD_REV64_MVT_v2f32_r(RetVT, Op0);
675 case MVT::v4f32: return fastEmit_AArch64ISD_REV64_MVT_v4f32_r(RetVT, Op0);
676 default: return Register();
677 }
678}
679
680// FastEmit functions for AArch64ISD::SADDLP.
681
682Register fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(MVT RetVT, Register Op0) {
683 if (RetVT.SimpleTy != MVT::v4i16)
684 return Register();
685 if ((Subtarget->isNeonAvailable())) {
686 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv8i8_v4i16, RC: &AArch64::FPR64RegClass, Op0);
687 }
688 return Register();
689}
690
691Register fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(MVT RetVT, Register Op0) {
692 if (RetVT.SimpleTy != MVT::v8i16)
693 return Register();
694 if ((Subtarget->isNeonAvailable())) {
695 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv16i8_v8i16, RC: &AArch64::FPR128RegClass, Op0);
696 }
697 return Register();
698}
699
700Register fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(MVT RetVT, Register Op0) {
701 if (RetVT.SimpleTy != MVT::v2i32)
702 return Register();
703 if ((Subtarget->isNeonAvailable())) {
704 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv4i16_v2i32, RC: &AArch64::FPR64RegClass, Op0);
705 }
706 return Register();
707}
708
709Register fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(MVT RetVT, Register Op0) {
710 if (RetVT.SimpleTy != MVT::v4i32)
711 return Register();
712 if ((Subtarget->isNeonAvailable())) {
713 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv8i16_v4i32, RC: &AArch64::FPR128RegClass, Op0);
714 }
715 return Register();
716}
717
718Register fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(MVT RetVT, Register Op0) {
719 if (RetVT.SimpleTy != MVT::v1i64)
720 return Register();
721 if ((Subtarget->isNeonAvailable())) {
722 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv2i32_v1i64, RC: &AArch64::FPR64RegClass, Op0);
723 }
724 return Register();
725}
726
727Register fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(MVT RetVT, Register Op0) {
728 if (RetVT.SimpleTy != MVT::v2i64)
729 return Register();
730 if ((Subtarget->isNeonAvailable())) {
731 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv4i32_v2i64, RC: &AArch64::FPR128RegClass, Op0);
732 }
733 return Register();
734}
735
736Register fastEmit_AArch64ISD_SADDLP_r(MVT VT, MVT RetVT, Register Op0) {
737 switch (VT.SimpleTy) {
738 case MVT::v8i8: return fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(RetVT, Op0);
739 case MVT::v16i8: return fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(RetVT, Op0);
740 case MVT::v4i16: return fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(RetVT, Op0);
741 case MVT::v8i16: return fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(RetVT, Op0);
742 case MVT::v2i32: return fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(RetVT, Op0);
743 case MVT::v4i32: return fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(RetVT, Op0);
744 default: return Register();
745 }
746}
747
748// FastEmit functions for AArch64ISD::SITOF.
749
750Register fastEmit_AArch64ISD_SITOF_MVT_f16_r(MVT RetVT, Register Op0) {
751 if (RetVT.SimpleTy != MVT::f16)
752 return Register();
753 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
754 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i16, RC: &AArch64::FPR16RegClass, Op0);
755 }
756 return Register();
757}
758
759Register fastEmit_AArch64ISD_SITOF_MVT_f32_r(MVT RetVT, Register Op0) {
760 if (RetVT.SimpleTy != MVT::f32)
761 return Register();
762 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
763 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i32, RC: &AArch64::FPR32RegClass, Op0);
764 }
765 return Register();
766}
767
768Register fastEmit_AArch64ISD_SITOF_MVT_f64_r(MVT RetVT, Register Op0) {
769 if (RetVT.SimpleTy != MVT::f64)
770 return Register();
771 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
772 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i64, RC: &AArch64::FPR64RegClass, Op0);
773 }
774 return Register();
775}
776
777Register fastEmit_AArch64ISD_SITOF_r(MVT VT, MVT RetVT, Register Op0) {
778 switch (VT.SimpleTy) {
779 case MVT::f16: return fastEmit_AArch64ISD_SITOF_MVT_f16_r(RetVT, Op0);
780 case MVT::f32: return fastEmit_AArch64ISD_SITOF_MVT_f32_r(RetVT, Op0);
781 case MVT::f64: return fastEmit_AArch64ISD_SITOF_MVT_f64_r(RetVT, Op0);
782 default: return Register();
783 }
784}
785
786// FastEmit functions for AArch64ISD::SQABS.
787
788Register fastEmit_AArch64ISD_SQABS_MVT_f32_r(MVT RetVT, Register Op0) {
789 if (RetVT.SimpleTy != MVT::f32)
790 return Register();
791 if ((Subtarget->isNeonAvailable())) {
792 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv1i32, RC: &AArch64::FPR32RegClass, Op0);
793 }
794 return Register();
795}
796
797Register fastEmit_AArch64ISD_SQABS_MVT_f64_r(MVT RetVT, Register Op0) {
798 if (RetVT.SimpleTy != MVT::f64)
799 return Register();
800 if ((Subtarget->isNeonAvailable())) {
801 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
802 }
803 return Register();
804}
805
806Register fastEmit_AArch64ISD_SQABS_MVT_v8i8_r(MVT RetVT, Register Op0) {
807 if (RetVT.SimpleTy != MVT::v8i8)
808 return Register();
809 if ((Subtarget->isNeonAvailable())) {
810 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv8i8, RC: &AArch64::FPR64RegClass, Op0);
811 }
812 return Register();
813}
814
815Register fastEmit_AArch64ISD_SQABS_MVT_v16i8_r(MVT RetVT, Register Op0) {
816 if (RetVT.SimpleTy != MVT::v16i8)
817 return Register();
818 if ((Subtarget->isNeonAvailable())) {
819 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv16i8, RC: &AArch64::FPR128RegClass, Op0);
820 }
821 return Register();
822}
823
824Register fastEmit_AArch64ISD_SQABS_MVT_v4i16_r(MVT RetVT, Register Op0) {
825 if (RetVT.SimpleTy != MVT::v4i16)
826 return Register();
827 if ((Subtarget->isNeonAvailable())) {
828 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv4i16, RC: &AArch64::FPR64RegClass, Op0);
829 }
830 return Register();
831}
832
833Register fastEmit_AArch64ISD_SQABS_MVT_v8i16_r(MVT RetVT, Register Op0) {
834 if (RetVT.SimpleTy != MVT::v8i16)
835 return Register();
836 if ((Subtarget->isNeonAvailable())) {
837 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv8i16, RC: &AArch64::FPR128RegClass, Op0);
838 }
839 return Register();
840}
841
842Register fastEmit_AArch64ISD_SQABS_MVT_v2i32_r(MVT RetVT, Register Op0) {
843 if (RetVT.SimpleTy != MVT::v2i32)
844 return Register();
845 if ((Subtarget->isNeonAvailable())) {
846 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv2i32, RC: &AArch64::FPR64RegClass, Op0);
847 }
848 return Register();
849}
850
851Register fastEmit_AArch64ISD_SQABS_MVT_v4i32_r(MVT RetVT, Register Op0) {
852 if (RetVT.SimpleTy != MVT::v4i32)
853 return Register();
854 if ((Subtarget->isNeonAvailable())) {
855 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv4i32, RC: &AArch64::FPR128RegClass, Op0);
856 }
857 return Register();
858}
859
860Register fastEmit_AArch64ISD_SQABS_MVT_v1i64_r(MVT RetVT, Register Op0) {
861 if (RetVT.SimpleTy != MVT::v1i64)
862 return Register();
863 if ((Subtarget->isNeonAvailable())) {
864 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
865 }
866 return Register();
867}
868
869Register fastEmit_AArch64ISD_SQABS_MVT_v2i64_r(MVT RetVT, Register Op0) {
870 if (RetVT.SimpleTy != MVT::v2i64)
871 return Register();
872 if ((Subtarget->isNeonAvailable())) {
873 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv2i64, RC: &AArch64::FPR128RegClass, Op0);
874 }
875 return Register();
876}
877
878Register fastEmit_AArch64ISD_SQABS_r(MVT VT, MVT RetVT, Register Op0) {
879 switch (VT.SimpleTy) {
880 case MVT::f32: return fastEmit_AArch64ISD_SQABS_MVT_f32_r(RetVT, Op0);
881 case MVT::f64: return fastEmit_AArch64ISD_SQABS_MVT_f64_r(RetVT, Op0);
882 case MVT::v8i8: return fastEmit_AArch64ISD_SQABS_MVT_v8i8_r(RetVT, Op0);
883 case MVT::v16i8: return fastEmit_AArch64ISD_SQABS_MVT_v16i8_r(RetVT, Op0);
884 case MVT::v4i16: return fastEmit_AArch64ISD_SQABS_MVT_v4i16_r(RetVT, Op0);
885 case MVT::v8i16: return fastEmit_AArch64ISD_SQABS_MVT_v8i16_r(RetVT, Op0);
886 case MVT::v2i32: return fastEmit_AArch64ISD_SQABS_MVT_v2i32_r(RetVT, Op0);
887 case MVT::v4i32: return fastEmit_AArch64ISD_SQABS_MVT_v4i32_r(RetVT, Op0);
888 case MVT::v1i64: return fastEmit_AArch64ISD_SQABS_MVT_v1i64_r(RetVT, Op0);
889 case MVT::v2i64: return fastEmit_AArch64ISD_SQABS_MVT_v2i64_r(RetVT, Op0);
890 default: return Register();
891 }
892}
893
894// FastEmit functions for AArch64ISD::SQNEG.
895
896Register fastEmit_AArch64ISD_SQNEG_MVT_f32_r(MVT RetVT, Register Op0) {
897 if (RetVT.SimpleTy != MVT::f32)
898 return Register();
899 if ((Subtarget->isNeonAvailable())) {
900 return fastEmitInst_r(MachineInstOpcode: AArch64::SQNEGv1i32, RC: &AArch64::FPR32RegClass, Op0);
901 }
902 return Register();
903}
904
905Register fastEmit_AArch64ISD_SQNEG_MVT_f64_r(MVT RetVT, Register Op0) {
906 if (RetVT.SimpleTy != MVT::f64)
907 return Register();
908 if ((Subtarget->isNeonAvailable())) {
909 return fastEmitInst_r(MachineInstOpcode: AArch64::SQNEGv1i64, RC: &AArch64::FPR64RegClass, Op0);
910 }
911 return Register();
912}
913
914Register fastEmit_AArch64ISD_SQNEG_r(MVT VT, MVT RetVT, Register Op0) {
915 switch (VT.SimpleTy) {
916 case MVT::f32: return fastEmit_AArch64ISD_SQNEG_MVT_f32_r(RetVT, Op0);
917 case MVT::f64: return fastEmit_AArch64ISD_SQNEG_MVT_f64_r(RetVT, Op0);
918 default: return Register();
919 }
920}
921
922// FastEmit functions for AArch64ISD::SUNPKHI.
923
924Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
925 if (RetVT.SimpleTy != MVT::nxv8i16)
926 return Register();
927 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
928 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
929 }
930 return Register();
931}
932
933Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
934 if (RetVT.SimpleTy != MVT::nxv4i32)
935 return Register();
936 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
937 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
938 }
939 return Register();
940}
941
942Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
943 if (RetVT.SimpleTy != MVT::nxv2i64)
944 return Register();
945 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
946 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
947 }
948 return Register();
949}
950
951Register fastEmit_AArch64ISD_SUNPKHI_r(MVT VT, MVT RetVT, Register Op0) {
952 switch (VT.SimpleTy) {
953 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(RetVT, Op0);
954 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(RetVT, Op0);
955 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(RetVT, Op0);
956 default: return Register();
957 }
958}
959
960// FastEmit functions for AArch64ISD::SUNPKLO.
961
962Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
963 if (RetVT.SimpleTy != MVT::nxv8i16)
964 return Register();
965 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
966 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
967 }
968 return Register();
969}
970
971Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
972 if (RetVT.SimpleTy != MVT::nxv4i32)
973 return Register();
974 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
975 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
976 }
977 return Register();
978}
979
980Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
981 if (RetVT.SimpleTy != MVT::nxv2i64)
982 return Register();
983 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
984 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
985 }
986 return Register();
987}
988
989Register fastEmit_AArch64ISD_SUNPKLO_r(MVT VT, MVT RetVT, Register Op0) {
990 switch (VT.SimpleTy) {
991 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(RetVT, Op0);
992 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(RetVT, Op0);
993 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(RetVT, Op0);
994 default: return Register();
995 }
996}
997
998// FastEmit functions for AArch64ISD::UADDLP.
999
1000Register fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(MVT RetVT, Register Op0) {
1001 if (RetVT.SimpleTy != MVT::v4i16)
1002 return Register();
1003 if ((Subtarget->isNeonAvailable())) {
1004 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv8i8_v4i16, RC: &AArch64::FPR64RegClass, Op0);
1005 }
1006 return Register();
1007}
1008
1009Register fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(MVT RetVT, Register Op0) {
1010 if (RetVT.SimpleTy != MVT::v8i16)
1011 return Register();
1012 if ((Subtarget->isNeonAvailable())) {
1013 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv16i8_v8i16, RC: &AArch64::FPR128RegClass, Op0);
1014 }
1015 return Register();
1016}
1017
1018Register fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(MVT RetVT, Register Op0) {
1019 if (RetVT.SimpleTy != MVT::v2i32)
1020 return Register();
1021 if ((Subtarget->isNeonAvailable())) {
1022 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv4i16_v2i32, RC: &AArch64::FPR64RegClass, Op0);
1023 }
1024 return Register();
1025}
1026
1027Register fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(MVT RetVT, Register Op0) {
1028 if (RetVT.SimpleTy != MVT::v4i32)
1029 return Register();
1030 if ((Subtarget->isNeonAvailable())) {
1031 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv8i16_v4i32, RC: &AArch64::FPR128RegClass, Op0);
1032 }
1033 return Register();
1034}
1035
1036Register fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(MVT RetVT, Register Op0) {
1037 if (RetVT.SimpleTy != MVT::v1i64)
1038 return Register();
1039 if ((Subtarget->isNeonAvailable())) {
1040 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv2i32_v1i64, RC: &AArch64::FPR64RegClass, Op0);
1041 }
1042 return Register();
1043}
1044
1045Register fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(MVT RetVT, Register Op0) {
1046 if (RetVT.SimpleTy != MVT::v2i64)
1047 return Register();
1048 if ((Subtarget->isNeonAvailable())) {
1049 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv4i32_v2i64, RC: &AArch64::FPR128RegClass, Op0);
1050 }
1051 return Register();
1052}
1053
1054Register fastEmit_AArch64ISD_UADDLP_r(MVT VT, MVT RetVT, Register Op0) {
1055 switch (VT.SimpleTy) {
1056 case MVT::v8i8: return fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(RetVT, Op0);
1057 case MVT::v16i8: return fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(RetVT, Op0);
1058 case MVT::v4i16: return fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(RetVT, Op0);
1059 case MVT::v8i16: return fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(RetVT, Op0);
1060 case MVT::v2i32: return fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(RetVT, Op0);
1061 case MVT::v4i32: return fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(RetVT, Op0);
1062 default: return Register();
1063 }
1064}
1065
1066// FastEmit functions for AArch64ISD::UITOF.
1067
1068Register fastEmit_AArch64ISD_UITOF_MVT_f16_r(MVT RetVT, Register Op0) {
1069 if (RetVT.SimpleTy != MVT::f16)
1070 return Register();
1071 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
1072 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i16, RC: &AArch64::FPR16RegClass, Op0);
1073 }
1074 return Register();
1075}
1076
1077Register fastEmit_AArch64ISD_UITOF_MVT_f32_r(MVT RetVT, Register Op0) {
1078 if (RetVT.SimpleTy != MVT::f32)
1079 return Register();
1080 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
1081 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i32, RC: &AArch64::FPR32RegClass, Op0);
1082 }
1083 return Register();
1084}
1085
1086Register fastEmit_AArch64ISD_UITOF_MVT_f64_r(MVT RetVT, Register Op0) {
1087 if (RetVT.SimpleTy != MVT::f64)
1088 return Register();
1089 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
1090 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i64, RC: &AArch64::FPR64RegClass, Op0);
1091 }
1092 return Register();
1093}
1094
1095Register fastEmit_AArch64ISD_UITOF_r(MVT VT, MVT RetVT, Register Op0) {
1096 switch (VT.SimpleTy) {
1097 case MVT::f16: return fastEmit_AArch64ISD_UITOF_MVT_f16_r(RetVT, Op0);
1098 case MVT::f32: return fastEmit_AArch64ISD_UITOF_MVT_f32_r(RetVT, Op0);
1099 case MVT::f64: return fastEmit_AArch64ISD_UITOF_MVT_f64_r(RetVT, Op0);
1100 default: return Register();
1101 }
1102}
1103
1104// FastEmit functions for AArch64ISD::UUNPKHI.
1105
1106Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
1107 if (RetVT.SimpleTy != MVT::nxv8i16)
1108 return Register();
1109 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1110 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1111 }
1112 return Register();
1113}
1114
1115Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
1116 if (RetVT.SimpleTy != MVT::nxv4i32)
1117 return Register();
1118 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1119 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1120 }
1121 return Register();
1122}
1123
1124Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
1125 if (RetVT.SimpleTy != MVT::nxv2i64)
1126 return Register();
1127 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1128 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1129 }
1130 return Register();
1131}
1132
1133Register fastEmit_AArch64ISD_UUNPKHI_r(MVT VT, MVT RetVT, Register Op0) {
1134 switch (VT.SimpleTy) {
1135 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(RetVT, Op0);
1136 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(RetVT, Op0);
1137 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(RetVT, Op0);
1138 default: return Register();
1139 }
1140}
1141
1142// FastEmit functions for AArch64ISD::UUNPKLO.
1143
1144Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
1145 if (RetVT.SimpleTy != MVT::nxv8i16)
1146 return Register();
1147 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1148 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1149 }
1150 return Register();
1151}
1152
1153Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
1154 if (RetVT.SimpleTy != MVT::nxv4i32)
1155 return Register();
1156 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1157 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1158 }
1159 return Register();
1160}
1161
1162Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
1163 if (RetVT.SimpleTy != MVT::nxv2i64)
1164 return Register();
1165 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1166 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1167 }
1168 return Register();
1169}
1170
1171Register fastEmit_AArch64ISD_UUNPKLO_r(MVT VT, MVT RetVT, Register Op0) {
1172 switch (VT.SimpleTy) {
1173 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(RetVT, Op0);
1174 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(RetVT, Op0);
1175 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(RetVT, Op0);
1176 default: return Register();
1177 }
1178}
1179
1180// FastEmit functions for ISD::ABS.
1181
1182Register fastEmit_ISD_ABS_MVT_i32_r(MVT RetVT, Register Op0) {
1183 if (RetVT.SimpleTy != MVT::i32)
1184 return Register();
1185 if ((Subtarget->hasCSSC())) {
1186 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSWr, RC: &AArch64::GPR32RegClass, Op0);
1187 }
1188 return Register();
1189}
1190
1191Register fastEmit_ISD_ABS_MVT_i64_r(MVT RetVT, Register Op0) {
1192 if (RetVT.SimpleTy != MVT::i64)
1193 return Register();
1194 if ((!Subtarget->hasCSSC())) {
1195 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
1196 }
1197 if ((Subtarget->hasCSSC())) {
1198 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSXr, RC: &AArch64::GPR64RegClass, Op0);
1199 }
1200 return Register();
1201}
1202
1203Register fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, Register Op0) {
1204 if (RetVT.SimpleTy != MVT::v8i8)
1205 return Register();
1206 if ((Subtarget->isNeonAvailable())) {
1207 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv8i8, RC: &AArch64::FPR64RegClass, Op0);
1208 }
1209 return Register();
1210}
1211
1212Register fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, Register Op0) {
1213 if (RetVT.SimpleTy != MVT::v16i8)
1214 return Register();
1215 if ((Subtarget->isNeonAvailable())) {
1216 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv16i8, RC: &AArch64::FPR128RegClass, Op0);
1217 }
1218 return Register();
1219}
1220
1221Register fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, Register Op0) {
1222 if (RetVT.SimpleTy != MVT::v4i16)
1223 return Register();
1224 if ((Subtarget->isNeonAvailable())) {
1225 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv4i16, RC: &AArch64::FPR64RegClass, Op0);
1226 }
1227 return Register();
1228}
1229
1230Register fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, Register Op0) {
1231 if (RetVT.SimpleTy != MVT::v8i16)
1232 return Register();
1233 if ((Subtarget->isNeonAvailable())) {
1234 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv8i16, RC: &AArch64::FPR128RegClass, Op0);
1235 }
1236 return Register();
1237}
1238
1239Register fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, Register Op0) {
1240 if (RetVT.SimpleTy != MVT::v2i32)
1241 return Register();
1242 if ((Subtarget->isNeonAvailable())) {
1243 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv2i32, RC: &AArch64::FPR64RegClass, Op0);
1244 }
1245 return Register();
1246}
1247
1248Register fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, Register Op0) {
1249 if (RetVT.SimpleTy != MVT::v4i32)
1250 return Register();
1251 if ((Subtarget->isNeonAvailable())) {
1252 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv4i32, RC: &AArch64::FPR128RegClass, Op0);
1253 }
1254 return Register();
1255}
1256
1257Register fastEmit_ISD_ABS_MVT_v1i64_r(MVT RetVT, Register Op0) {
1258 if (RetVT.SimpleTy != MVT::v1i64)
1259 return Register();
1260 if ((Subtarget->isNeonAvailable())) {
1261 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
1262 }
1263 return Register();
1264}
1265
1266Register fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, Register Op0) {
1267 if (RetVT.SimpleTy != MVT::v2i64)
1268 return Register();
1269 if ((Subtarget->isNeonAvailable())) {
1270 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv2i64, RC: &AArch64::FPR128RegClass, Op0);
1271 }
1272 return Register();
1273}
1274
1275Register fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, Register Op0) {
1276 switch (VT.SimpleTy) {
1277 case MVT::i32: return fastEmit_ISD_ABS_MVT_i32_r(RetVT, Op0);
1278 case MVT::i64: return fastEmit_ISD_ABS_MVT_i64_r(RetVT, Op0);
1279 case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0);
1280 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
1281 case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0);
1282 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
1283 case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0);
1284 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
1285 case MVT::v1i64: return fastEmit_ISD_ABS_MVT_v1i64_r(RetVT, Op0);
1286 case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0);
1287 default: return Register();
1288 }
1289}
1290
1291// FastEmit functions for ISD::BITCAST.
1292
1293Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Register Op0) {
1294 if ((!Subtarget->isLittleEndian())) {
1295 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1296 }
1297 return Register();
1298}
1299
1300Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Register Op0) {
1301 if ((!Subtarget->isLittleEndian())) {
1302 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1303 }
1304 return Register();
1305}
1306
1307Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Register Op0) {
1308 if ((!Subtarget->isLittleEndian())) {
1309 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1310 }
1311 return Register();
1312}
1313
1314Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Register Op0) {
1315 if ((!Subtarget->isLittleEndian())) {
1316 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1317 }
1318 return Register();
1319}
1320
1321Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Register Op0) {
1322 if ((!Subtarget->isLittleEndian())) {
1323 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1324 }
1325 return Register();
1326}
1327
1328Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Register Op0) {
1329 if ((!Subtarget->isLittleEndian())) {
1330 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1331 }
1332 return Register();
1333}
1334
1335Register fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, Register Op0) {
1336switch (RetVT.SimpleTy) {
1337 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0);
1338 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0);
1339 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0);
1340 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0);
1341 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Op0);
1342 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0);
1343 default: return Register();
1344}
1345}
1346
1347Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Register Op0) {
1348 if ((!Subtarget->isLittleEndian())) {
1349 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1350 }
1351 return Register();
1352}
1353
1354Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Register Op0) {
1355 if ((!Subtarget->isLittleEndian())) {
1356 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1357 }
1358 return Register();
1359}
1360
1361Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Register Op0) {
1362 if ((!Subtarget->isLittleEndian())) {
1363 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1364 }
1365 return Register();
1366}
1367
1368Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Register Op0) {
1369 if ((!Subtarget->isLittleEndian())) {
1370 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1371 }
1372 return Register();
1373}
1374
1375Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Register Op0) {
1376 if ((!Subtarget->isLittleEndian())) {
1377 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1378 }
1379 return Register();
1380}
1381
1382Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Register Op0) {
1383 if ((!Subtarget->isLittleEndian())) {
1384 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1385 }
1386 return Register();
1387}
1388
1389Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Register Op0) {
1390 if ((!Subtarget->isLittleEndian())) {
1391 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1392 }
1393 return Register();
1394}
1395
1396Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Register Op0) {
1397 if ((!Subtarget->isLittleEndian())) {
1398 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1399 }
1400 return Register();
1401}
1402
1403Register fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, Register Op0) {
1404switch (RetVT.SimpleTy) {
1405 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0);
1406 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0);
1407 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0);
1408 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0);
1409 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0);
1410 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Op0);
1411 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0);
1412 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Op0);
1413 default: return Register();
1414}
1415}
1416
1417Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Register Op0) {
1418 if ((!Subtarget->isLittleEndian())) {
1419 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1420 }
1421 return Register();
1422}
1423
1424Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Register Op0) {
1425 if ((!Subtarget->isLittleEndian())) {
1426 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1427 }
1428 return Register();
1429}
1430
1431Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Register Op0) {
1432 if ((!Subtarget->isLittleEndian())) {
1433 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1434 }
1435 return Register();
1436}
1437
1438Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Register Op0) {
1439 if ((!Subtarget->isLittleEndian())) {
1440 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1441 }
1442 return Register();
1443}
1444
1445Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Register Op0) {
1446 if ((!Subtarget->isLittleEndian())) {
1447 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1448 }
1449 return Register();
1450}
1451
1452Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Register Op0) {
1453 if ((!Subtarget->isLittleEndian())) {
1454 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1455 }
1456 return Register();
1457}
1458
1459Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Register Op0) {
1460 if ((!Subtarget->isLittleEndian())) {
1461 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1462 }
1463 return Register();
1464}
1465
1466Register fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, Register Op0) {
1467switch (RetVT.SimpleTy) {
1468 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0);
1469 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0);
1470 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0);
1471 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0);
1472 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Op0);
1473 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0);
1474 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0);
1475 default: return Register();
1476}
1477}
1478
1479Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Register Op0) {
1480 if ((!Subtarget->isLittleEndian())) {
1481 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1482 }
1483 return Register();
1484}
1485
1486Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Register Op0) {
1487 if ((!Subtarget->isLittleEndian())) {
1488 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1489 }
1490 return Register();
1491}
1492
1493Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Register Op0) {
1494 if ((!Subtarget->isLittleEndian())) {
1495 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1496 }
1497 return Register();
1498}
1499
1500Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Register Op0) {
1501 if ((!Subtarget->isLittleEndian())) {
1502 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1503 }
1504 return Register();
1505}
1506
1507Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Register Op0) {
1508 if ((!Subtarget->isLittleEndian())) {
1509 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1510 }
1511 return Register();
1512}
1513
1514Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Register Op0) {
1515 if ((!Subtarget->isLittleEndian())) {
1516 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1517 }
1518 return Register();
1519}
1520
1521Register fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, Register Op0) {
1522switch (RetVT.SimpleTy) {
1523 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0);
1524 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0);
1525 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0);
1526 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0);
1527 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0);
1528 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Op0);
1529 default: return Register();
1530}
1531}
1532
1533Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Register Op0) {
1534 if ((!Subtarget->isLittleEndian())) {
1535 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1536 }
1537 return Register();
1538}
1539
1540Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Register Op0) {
1541 if ((!Subtarget->isLittleEndian())) {
1542 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1543 }
1544 return Register();
1545}
1546
1547Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Register Op0) {
1548 if ((!Subtarget->isLittleEndian())) {
1549 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1550 }
1551 return Register();
1552}
1553
1554Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Register Op0) {
1555 if ((!Subtarget->isLittleEndian())) {
1556 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1557 }
1558 return Register();
1559}
1560
1561Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Register Op0) {
1562 if ((!Subtarget->isLittleEndian())) {
1563 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1564 }
1565 return Register();
1566}
1567
1568Register fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, Register Op0) {
1569switch (RetVT.SimpleTy) {
1570 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0);
1571 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0);
1572 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0);
1573 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0);
1574 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0);
1575 default: return Register();
1576}
1577}
1578
1579Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Register Op0) {
1580 if ((!Subtarget->isLittleEndian())) {
1581 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1582 }
1583 return Register();
1584}
1585
1586Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Register Op0) {
1587 if ((!Subtarget->isLittleEndian())) {
1588 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1589 }
1590 return Register();
1591}
1592
1593Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Register Op0) {
1594 if ((!Subtarget->isLittleEndian())) {
1595 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1596 }
1597 return Register();
1598}
1599
1600Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Register Op0) {
1601 if ((!Subtarget->isLittleEndian())) {
1602 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1603 }
1604 return Register();
1605}
1606
1607Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Register Op0) {
1608 if ((!Subtarget->isLittleEndian())) {
1609 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1610 }
1611 return Register();
1612}
1613
1614Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Register Op0) {
1615 if ((!Subtarget->isLittleEndian())) {
1616 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1617 }
1618 return Register();
1619}
1620
1621Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Register Op0) {
1622 if ((!Subtarget->isLittleEndian())) {
1623 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1624 }
1625 return Register();
1626}
1627
1628Register fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, Register Op0) {
1629switch (RetVT.SimpleTy) {
1630 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0);
1631 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0);
1632 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0);
1633 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0);
1634 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0);
1635 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Op0);
1636 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Op0);
1637 default: return Register();
1638}
1639}
1640
1641Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Register Op0) {
1642 if ((!Subtarget->isLittleEndian())) {
1643 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1644 }
1645 return Register();
1646}
1647
1648Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Register Op0) {
1649 if ((!Subtarget->isLittleEndian())) {
1650 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1651 }
1652 return Register();
1653}
1654
1655Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Register Op0) {
1656 if ((!Subtarget->isLittleEndian())) {
1657 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1658 }
1659 return Register();
1660}
1661
1662Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Register Op0) {
1663 if ((!Subtarget->isLittleEndian())) {
1664 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1665 }
1666 return Register();
1667}
1668
1669Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Register Op0) {
1670 if ((!Subtarget->isLittleEndian())) {
1671 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1672 }
1673 return Register();
1674}
1675
1676Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Register Op0) {
1677 if ((!Subtarget->isLittleEndian())) {
1678 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1679 }
1680 return Register();
1681}
1682
1683Register fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, Register Op0) {
1684switch (RetVT.SimpleTy) {
1685 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0);
1686 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0);
1687 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0);
1688 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0);
1689 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Op0);
1690 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0);
1691 default: return Register();
1692}
1693}
1694
1695Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Register Op0) {
1696 if ((!Subtarget->isLittleEndian())) {
1697 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1698 }
1699 return Register();
1700}
1701
1702Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Register Op0) {
1703 if ((!Subtarget->isLittleEndian())) {
1704 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1705 }
1706 return Register();
1707}
1708
1709Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Register Op0) {
1710 if ((!Subtarget->isLittleEndian())) {
1711 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1712 }
1713 return Register();
1714}
1715
1716Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Register Op0) {
1717 if ((!Subtarget->isLittleEndian())) {
1718 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1719 }
1720 return Register();
1721}
1722
1723Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Register Op0) {
1724 if ((!Subtarget->isLittleEndian())) {
1725 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1726 }
1727 return Register();
1728}
1729
1730Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Register Op0) {
1731 if ((!Subtarget->isLittleEndian())) {
1732 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1733 }
1734 return Register();
1735}
1736
1737Register fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, Register Op0) {
1738switch (RetVT.SimpleTy) {
1739 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0);
1740 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0);
1741 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0);
1742 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0);
1743 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Op0);
1744 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0);
1745 default: return Register();
1746}
1747}
1748
1749Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Register Op0) {
1750 if ((!Subtarget->isLittleEndian())) {
1751 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1752 }
1753 return Register();
1754}
1755
1756Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Register Op0) {
1757 if ((!Subtarget->isLittleEndian())) {
1758 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1759 }
1760 return Register();
1761}
1762
1763Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Register Op0) {
1764 if ((!Subtarget->isLittleEndian())) {
1765 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1766 }
1767 return Register();
1768}
1769
1770Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Register Op0) {
1771 if ((!Subtarget->isLittleEndian())) {
1772 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1773 }
1774 return Register();
1775}
1776
1777Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Register Op0) {
1778 if ((!Subtarget->isLittleEndian())) {
1779 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1780 }
1781 return Register();
1782}
1783
1784Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Register Op0) {
1785 if ((!Subtarget->isLittleEndian())) {
1786 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1787 }
1788 return Register();
1789}
1790
1791Register fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, Register Op0) {
1792switch (RetVT.SimpleTy) {
1793 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0);
1794 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0);
1795 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0);
1796 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0);
1797 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Op0);
1798 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0);
1799 default: return Register();
1800}
1801}
1802
1803Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Register Op0) {
1804 if ((!Subtarget->isLittleEndian())) {
1805 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1806 }
1807 return Register();
1808}
1809
1810Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Register Op0) {
1811 if ((!Subtarget->isLittleEndian())) {
1812 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1813 }
1814 return Register();
1815}
1816
1817Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Register Op0) {
1818 if ((!Subtarget->isLittleEndian())) {
1819 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1820 }
1821 return Register();
1822}
1823
1824Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Register Op0) {
1825 if ((!Subtarget->isLittleEndian())) {
1826 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1827 }
1828 return Register();
1829}
1830
1831Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Register Op0) {
1832 if ((!Subtarget->isLittleEndian())) {
1833 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1834 }
1835 return Register();
1836}
1837
1838Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Register Op0) {
1839 if ((!Subtarget->isLittleEndian())) {
1840 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1841 }
1842 return Register();
1843}
1844
1845Register fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, Register Op0) {
1846switch (RetVT.SimpleTy) {
1847 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0);
1848 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0);
1849 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0);
1850 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0);
1851 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0);
1852 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Op0);
1853 default: return Register();
1854}
1855}
1856
1857Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Register Op0) {
1858 if ((!Subtarget->isLittleEndian())) {
1859 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1860 }
1861 return Register();
1862}
1863
1864Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Register Op0) {
1865 if ((!Subtarget->isLittleEndian())) {
1866 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1867 }
1868 return Register();
1869}
1870
1871Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Register Op0) {
1872 if ((!Subtarget->isLittleEndian())) {
1873 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1874 }
1875 return Register();
1876}
1877
1878Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Register Op0) {
1879 if ((!Subtarget->isLittleEndian())) {
1880 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1881 }
1882 return Register();
1883}
1884
1885Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Register Op0) {
1886 if ((!Subtarget->isLittleEndian())) {
1887 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1888 }
1889 return Register();
1890}
1891
1892Register fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, Register Op0) {
1893switch (RetVT.SimpleTy) {
1894 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0);
1895 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0);
1896 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0);
1897 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0);
1898 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0);
1899 default: return Register();
1900}
1901}
1902
1903Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Register Op0) {
1904 if ((!Subtarget->isLittleEndian())) {
1905 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1906 }
1907 return Register();
1908}
1909
1910Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Register Op0) {
1911 if ((!Subtarget->isLittleEndian())) {
1912 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1913 }
1914 return Register();
1915}
1916
1917Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Register Op0) {
1918 if ((!Subtarget->isLittleEndian())) {
1919 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1920 }
1921 return Register();
1922}
1923
1924Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Register Op0) {
1925 if ((!Subtarget->isLittleEndian())) {
1926 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1927 }
1928 return Register();
1929}
1930
1931Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Register Op0) {
1932 if ((!Subtarget->isLittleEndian())) {
1933 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1934 }
1935 return Register();
1936}
1937
1938Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(Register Op0) {
1939 if ((!Subtarget->isLittleEndian())) {
1940 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1941 }
1942 return Register();
1943}
1944
1945Register fastEmit_ISD_BITCAST_MVT_v4bf16_r(MVT RetVT, Register Op0) {
1946switch (RetVT.SimpleTy) {
1947 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Op0);
1948 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Op0);
1949 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Op0);
1950 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Op0);
1951 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Op0);
1952 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(Op0);
1953 default: return Register();
1954}
1955}
1956
1957Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Register Op0) {
1958 if ((!Subtarget->isLittleEndian())) {
1959 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1960 }
1961 return Register();
1962}
1963
1964Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Register Op0) {
1965 if ((!Subtarget->isLittleEndian())) {
1966 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1967 }
1968 return Register();
1969}
1970
1971Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Register Op0) {
1972 if ((!Subtarget->isLittleEndian())) {
1973 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1974 }
1975 return Register();
1976}
1977
1978Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Register Op0) {
1979 if ((!Subtarget->isLittleEndian())) {
1980 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1981 }
1982 return Register();
1983}
1984
1985Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Register Op0) {
1986 if ((!Subtarget->isLittleEndian())) {
1987 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1988 }
1989 return Register();
1990}
1991
1992Register fastEmit_ISD_BITCAST_MVT_v8bf16_r(MVT RetVT, Register Op0) {
1993switch (RetVT.SimpleTy) {
1994 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Op0);
1995 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Op0);
1996 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Op0);
1997 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Op0);
1998 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Op0);
1999 default: return Register();
2000}
2001}
2002
2003Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Register Op0) {
2004 if ((!Subtarget->isLittleEndian())) {
2005 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2006 }
2007 return Register();
2008}
2009
2010Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Register Op0) {
2011 if ((!Subtarget->isLittleEndian())) {
2012 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
2013 }
2014 return Register();
2015}
2016
2017Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Register Op0) {
2018 if ((!Subtarget->isLittleEndian())) {
2019 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2020 }
2021 return Register();
2022}
2023
2024Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Register Op0) {
2025 if ((!Subtarget->isLittleEndian())) {
2026 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2027 }
2028 return Register();
2029}
2030
2031Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Register Op0) {
2032 if ((!Subtarget->isLittleEndian())) {
2033 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2034 }
2035 return Register();
2036}
2037
2038Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Register Op0) {
2039 if ((!Subtarget->isLittleEndian())) {
2040 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2041 }
2042 return Register();
2043}
2044
2045Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Register Op0) {
2046 if ((!Subtarget->isLittleEndian())) {
2047 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2048 }
2049 return Register();
2050}
2051
2052Register fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, Register Op0) {
2053switch (RetVT.SimpleTy) {
2054 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0);
2055 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0);
2056 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0);
2057 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0);
2058 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0);
2059 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Op0);
2060 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Op0);
2061 default: return Register();
2062}
2063}
2064
2065Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Register Op0) {
2066 if ((!Subtarget->isLittleEndian())) {
2067 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
2068 }
2069 return Register();
2070}
2071
2072Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Register Op0) {
2073 if ((!Subtarget->isLittleEndian())) {
2074 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2075 }
2076 return Register();
2077}
2078
2079Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Register Op0) {
2080 if ((!Subtarget->isLittleEndian())) {
2081 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2082 }
2083 return Register();
2084}
2085
2086Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Register Op0) {
2087 if ((!Subtarget->isLittleEndian())) {
2088 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2089 }
2090 return Register();
2091}
2092
2093Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Register Op0) {
2094 if ((!Subtarget->isLittleEndian())) {
2095 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2096 }
2097 return Register();
2098}
2099
2100Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Register Op0) {
2101 if ((!Subtarget->isLittleEndian())) {
2102 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2103 }
2104 return Register();
2105}
2106
2107Register fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, Register Op0) {
2108switch (RetVT.SimpleTy) {
2109 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0);
2110 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0);
2111 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0);
2112 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0);
2113 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Op0);
2114 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0);
2115 default: return Register();
2116}
2117}
2118
2119Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Register Op0) {
2120 if ((!Subtarget->isLittleEndian())) {
2121 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
2122 }
2123 return Register();
2124}
2125
2126Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Register Op0) {
2127 if ((!Subtarget->isLittleEndian())) {
2128 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2129 }
2130 return Register();
2131}
2132
2133Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Register Op0) {
2134 if ((!Subtarget->isLittleEndian())) {
2135 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2136 }
2137 return Register();
2138}
2139
2140Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Register Op0) {
2141 if ((!Subtarget->isLittleEndian())) {
2142 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2143 }
2144 return Register();
2145}
2146
2147Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(Register Op0) {
2148 if ((!Subtarget->isLittleEndian())) {
2149 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2150 }
2151 return Register();
2152}
2153
2154Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Register Op0) {
2155 if ((!Subtarget->isLittleEndian())) {
2156 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2157 }
2158 return Register();
2159}
2160
2161Register fastEmit_ISD_BITCAST_MVT_v1f64_r(MVT RetVT, Register Op0) {
2162switch (RetVT.SimpleTy) {
2163 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Op0);
2164 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Op0);
2165 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Op0);
2166 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Op0);
2167 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(Op0);
2168 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Op0);
2169 default: return Register();
2170}
2171}
2172
2173Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Register Op0) {
2174 if ((!Subtarget->isLittleEndian())) {
2175 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2176 }
2177 return Register();
2178}
2179
2180Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Register Op0) {
2181 if ((!Subtarget->isLittleEndian())) {
2182 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2183 }
2184 return Register();
2185}
2186
2187Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Register Op0) {
2188 if ((!Subtarget->isLittleEndian())) {
2189 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2190 }
2191 return Register();
2192}
2193
2194Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Register Op0) {
2195 if ((!Subtarget->isLittleEndian())) {
2196 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2197 }
2198 return Register();
2199}
2200
2201Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Register Op0) {
2202 if ((!Subtarget->isLittleEndian())) {
2203 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2204 }
2205 return Register();
2206}
2207
2208Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Register Op0) {
2209 if ((!Subtarget->isLittleEndian())) {
2210 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2211 }
2212 return Register();
2213}
2214
2215Register fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, Register Op0) {
2216switch (RetVT.SimpleTy) {
2217 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0);
2218 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0);
2219 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0);
2220 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0);
2221 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Op0);
2222 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0);
2223 default: return Register();
2224}
2225}
2226
2227Register fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, Register Op0) {
2228 switch (VT.SimpleTy) {
2229 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
2230 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0);
2231 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0);
2232 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0);
2233 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0);
2234 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0);
2235 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0);
2236 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0);
2237 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0);
2238 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0);
2239 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0);
2240 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v4bf16_r(RetVT, Op0);
2241 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v8bf16_r(RetVT, Op0);
2242 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0);
2243 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0);
2244 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v1f64_r(RetVT, Op0);
2245 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0);
2246 default: return Register();
2247 }
2248}
2249
2250// FastEmit functions for ISD::BITREVERSE.
2251
2252Register fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, Register Op0) {
2253 if (RetVT.SimpleTy != MVT::i32)
2254 return Register();
2255 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITWr, RC: &AArch64::GPR32RegClass, Op0);
2256}
2257
2258Register fastEmit_ISD_BITREVERSE_MVT_i64_r(MVT RetVT, Register Op0) {
2259 if (RetVT.SimpleTy != MVT::i64)
2260 return Register();
2261 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITXr, RC: &AArch64::GPR64RegClass, Op0);
2262}
2263
2264Register fastEmit_ISD_BITREVERSE_MVT_v8i8_r(MVT RetVT, Register Op0) {
2265 if (RetVT.SimpleTy != MVT::v8i8)
2266 return Register();
2267 if ((Subtarget->isNeonAvailable())) {
2268 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITv8i8, RC: &AArch64::FPR64RegClass, Op0);
2269 }
2270 return Register();
2271}
2272
2273Register fastEmit_ISD_BITREVERSE_MVT_v16i8_r(MVT RetVT, Register Op0) {
2274 if (RetVT.SimpleTy != MVT::v16i8)
2275 return Register();
2276 if ((Subtarget->isNeonAvailable())) {
2277 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITv16i8, RC: &AArch64::FPR128RegClass, Op0);
2278 }
2279 return Register();
2280}
2281
2282Register fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, Register Op0) {
2283 switch (VT.SimpleTy) {
2284 case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0);
2285 case MVT::i64: return fastEmit_ISD_BITREVERSE_MVT_i64_r(RetVT, Op0);
2286 case MVT::v8i8: return fastEmit_ISD_BITREVERSE_MVT_v8i8_r(RetVT, Op0);
2287 case MVT::v16i8: return fastEmit_ISD_BITREVERSE_MVT_v16i8_r(RetVT, Op0);
2288 default: return Register();
2289 }
2290}
2291
2292// FastEmit functions for ISD::BSWAP.
2293
2294Register fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, Register Op0) {
2295 if (RetVT.SimpleTy != MVT::i32)
2296 return Register();
2297 return fastEmitInst_r(MachineInstOpcode: AArch64::REVWr, RC: &AArch64::GPR32RegClass, Op0);
2298}
2299
2300Register fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, Register Op0) {
2301 if (RetVT.SimpleTy != MVT::i64)
2302 return Register();
2303 return fastEmitInst_r(MachineInstOpcode: AArch64::REVXr, RC: &AArch64::GPR64RegClass, Op0);
2304}
2305
2306Register fastEmit_ISD_BSWAP_MVT_v4i16_r(MVT RetVT, Register Op0) {
2307 if (RetVT.SimpleTy != MVT::v4i16)
2308 return Register();
2309 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
2310}
2311
2312Register fastEmit_ISD_BSWAP_MVT_v8i16_r(MVT RetVT, Register Op0) {
2313 if (RetVT.SimpleTy != MVT::v8i16)
2314 return Register();
2315 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
2316}
2317
2318Register fastEmit_ISD_BSWAP_MVT_v2i32_r(MVT RetVT, Register Op0) {
2319 if (RetVT.SimpleTy != MVT::v2i32)
2320 return Register();
2321 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
2322}
2323
2324Register fastEmit_ISD_BSWAP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2325 if (RetVT.SimpleTy != MVT::v4i32)
2326 return Register();
2327 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
2328}
2329
2330Register fastEmit_ISD_BSWAP_MVT_v2i64_r(MVT RetVT, Register Op0) {
2331 if (RetVT.SimpleTy != MVT::v2i64)
2332 return Register();
2333 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2334}
2335
2336Register fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, Register Op0) {
2337 switch (VT.SimpleTy) {
2338 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
2339 case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0);
2340 case MVT::v4i16: return fastEmit_ISD_BSWAP_MVT_v4i16_r(RetVT, Op0);
2341 case MVT::v8i16: return fastEmit_ISD_BSWAP_MVT_v8i16_r(RetVT, Op0);
2342 case MVT::v2i32: return fastEmit_ISD_BSWAP_MVT_v2i32_r(RetVT, Op0);
2343 case MVT::v4i32: return fastEmit_ISD_BSWAP_MVT_v4i32_r(RetVT, Op0);
2344 case MVT::v2i64: return fastEmit_ISD_BSWAP_MVT_v2i64_r(RetVT, Op0);
2345 default: return Register();
2346 }
2347}
2348
2349// FastEmit functions for ISD::CTLS.
2350
2351Register fastEmit_ISD_CTLS_MVT_i32_r(MVT RetVT, Register Op0) {
2352 if (RetVT.SimpleTy != MVT::i32)
2353 return Register();
2354 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSWr, RC: &AArch64::GPR32RegClass, Op0);
2355}
2356
2357Register fastEmit_ISD_CTLS_MVT_i64_r(MVT RetVT, Register Op0) {
2358 if (RetVT.SimpleTy != MVT::i64)
2359 return Register();
2360 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSXr, RC: &AArch64::GPR64RegClass, Op0);
2361}
2362
2363Register fastEmit_ISD_CTLS_MVT_v8i8_r(MVT RetVT, Register Op0) {
2364 if (RetVT.SimpleTy != MVT::v8i8)
2365 return Register();
2366 if ((Subtarget->isNeonAvailable())) {
2367 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv8i8, RC: &AArch64::FPR64RegClass, Op0);
2368 }
2369 return Register();
2370}
2371
2372Register fastEmit_ISD_CTLS_MVT_v16i8_r(MVT RetVT, Register Op0) {
2373 if (RetVT.SimpleTy != MVT::v16i8)
2374 return Register();
2375 if ((Subtarget->isNeonAvailable())) {
2376 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv16i8, RC: &AArch64::FPR128RegClass, Op0);
2377 }
2378 return Register();
2379}
2380
2381Register fastEmit_ISD_CTLS_MVT_v4i16_r(MVT RetVT, Register Op0) {
2382 if (RetVT.SimpleTy != MVT::v4i16)
2383 return Register();
2384 if ((Subtarget->isNeonAvailable())) {
2385 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv4i16, RC: &AArch64::FPR64RegClass, Op0);
2386 }
2387 return Register();
2388}
2389
2390Register fastEmit_ISD_CTLS_MVT_v8i16_r(MVT RetVT, Register Op0) {
2391 if (RetVT.SimpleTy != MVT::v8i16)
2392 return Register();
2393 if ((Subtarget->isNeonAvailable())) {
2394 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv8i16, RC: &AArch64::FPR128RegClass, Op0);
2395 }
2396 return Register();
2397}
2398
2399Register fastEmit_ISD_CTLS_MVT_v2i32_r(MVT RetVT, Register Op0) {
2400 if (RetVT.SimpleTy != MVT::v2i32)
2401 return Register();
2402 if ((Subtarget->isNeonAvailable())) {
2403 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv2i32, RC: &AArch64::FPR64RegClass, Op0);
2404 }
2405 return Register();
2406}
2407
2408Register fastEmit_ISD_CTLS_MVT_v4i32_r(MVT RetVT, Register Op0) {
2409 if (RetVT.SimpleTy != MVT::v4i32)
2410 return Register();
2411 if ((Subtarget->isNeonAvailable())) {
2412 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv4i32, RC: &AArch64::FPR128RegClass, Op0);
2413 }
2414 return Register();
2415}
2416
2417Register fastEmit_ISD_CTLS_r(MVT VT, MVT RetVT, Register Op0) {
2418 switch (VT.SimpleTy) {
2419 case MVT::i32: return fastEmit_ISD_CTLS_MVT_i32_r(RetVT, Op0);
2420 case MVT::i64: return fastEmit_ISD_CTLS_MVT_i64_r(RetVT, Op0);
2421 case MVT::v8i8: return fastEmit_ISD_CTLS_MVT_v8i8_r(RetVT, Op0);
2422 case MVT::v16i8: return fastEmit_ISD_CTLS_MVT_v16i8_r(RetVT, Op0);
2423 case MVT::v4i16: return fastEmit_ISD_CTLS_MVT_v4i16_r(RetVT, Op0);
2424 case MVT::v8i16: return fastEmit_ISD_CTLS_MVT_v8i16_r(RetVT, Op0);
2425 case MVT::v2i32: return fastEmit_ISD_CTLS_MVT_v2i32_r(RetVT, Op0);
2426 case MVT::v4i32: return fastEmit_ISD_CTLS_MVT_v4i32_r(RetVT, Op0);
2427 default: return Register();
2428 }
2429}
2430
2431// FastEmit functions for ISD::CTLZ.
2432
2433Register fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, Register Op0) {
2434 if (RetVT.SimpleTy != MVT::i32)
2435 return Register();
2436 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZWr, RC: &AArch64::GPR32RegClass, Op0);
2437}
2438
2439Register fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, Register Op0) {
2440 if (RetVT.SimpleTy != MVT::i64)
2441 return Register();
2442 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZXr, RC: &AArch64::GPR64RegClass, Op0);
2443}
2444
2445Register fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, Register Op0) {
2446 if (RetVT.SimpleTy != MVT::v8i8)
2447 return Register();
2448 if ((Subtarget->isNeonAvailable())) {
2449 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv8i8, RC: &AArch64::FPR64RegClass, Op0);
2450 }
2451 return Register();
2452}
2453
2454Register fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, Register Op0) {
2455 if (RetVT.SimpleTy != MVT::v16i8)
2456 return Register();
2457 if ((Subtarget->isNeonAvailable())) {
2458 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv16i8, RC: &AArch64::FPR128RegClass, Op0);
2459 }
2460 return Register();
2461}
2462
2463Register fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, Register Op0) {
2464 if (RetVT.SimpleTy != MVT::v4i16)
2465 return Register();
2466 if ((Subtarget->isNeonAvailable())) {
2467 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv4i16, RC: &AArch64::FPR64RegClass, Op0);
2468 }
2469 return Register();
2470}
2471
2472Register fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, Register Op0) {
2473 if (RetVT.SimpleTy != MVT::v8i16)
2474 return Register();
2475 if ((Subtarget->isNeonAvailable())) {
2476 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv8i16, RC: &AArch64::FPR128RegClass, Op0);
2477 }
2478 return Register();
2479}
2480
2481Register fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, Register Op0) {
2482 if (RetVT.SimpleTy != MVT::v2i32)
2483 return Register();
2484 if ((Subtarget->isNeonAvailable())) {
2485 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv2i32, RC: &AArch64::FPR64RegClass, Op0);
2486 }
2487 return Register();
2488}
2489
2490Register fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, Register Op0) {
2491 if (RetVT.SimpleTy != MVT::v4i32)
2492 return Register();
2493 if ((Subtarget->isNeonAvailable())) {
2494 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv4i32, RC: &AArch64::FPR128RegClass, Op0);
2495 }
2496 return Register();
2497}
2498
2499Register fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, Register Op0) {
2500 switch (VT.SimpleTy) {
2501 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
2502 case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0);
2503 case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0);
2504 case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0);
2505 case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0);
2506 case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0);
2507 case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0);
2508 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
2509 default: return Register();
2510 }
2511}
2512
2513// FastEmit functions for ISD::CTPOP.
2514
2515Register fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, Register Op0) {
2516 if (RetVT.SimpleTy != MVT::i32)
2517 return Register();
2518 if ((Subtarget->hasCSSC())) {
2519 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTWr, RC: &AArch64::GPR32RegClass, Op0);
2520 }
2521 return Register();
2522}
2523
2524Register fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, Register Op0) {
2525 if (RetVT.SimpleTy != MVT::i64)
2526 return Register();
2527 if ((Subtarget->hasCSSC())) {
2528 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTXr, RC: &AArch64::GPR64RegClass, Op0);
2529 }
2530 return Register();
2531}
2532
2533Register fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, Register Op0) {
2534 if (RetVT.SimpleTy != MVT::v8i8)
2535 return Register();
2536 if ((Subtarget->isNeonAvailable())) {
2537 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTv8i8, RC: &AArch64::FPR64RegClass, Op0);
2538 }
2539 return Register();
2540}
2541
2542Register fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, Register Op0) {
2543 if (RetVT.SimpleTy != MVT::v16i8)
2544 return Register();
2545 if ((Subtarget->isNeonAvailable())) {
2546 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTv16i8, RC: &AArch64::FPR128RegClass, Op0);
2547 }
2548 return Register();
2549}
2550
2551Register fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, Register Op0) {
2552 switch (VT.SimpleTy) {
2553 case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0);
2554 case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0);
2555 case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0);
2556 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
2557 default: return Register();
2558 }
2559}
2560
2561// FastEmit functions for ISD::CTTZ.
2562
2563Register fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, Register Op0) {
2564 if (RetVT.SimpleTy != MVT::i32)
2565 return Register();
2566 if ((Subtarget->hasCSSC())) {
2567 return fastEmitInst_r(MachineInstOpcode: AArch64::CTZWr, RC: &AArch64::GPR32RegClass, Op0);
2568 }
2569 return Register();
2570}
2571
2572Register fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, Register Op0) {
2573 if (RetVT.SimpleTy != MVT::i64)
2574 return Register();
2575 if ((Subtarget->hasCSSC())) {
2576 return fastEmitInst_r(MachineInstOpcode: AArch64::CTZXr, RC: &AArch64::GPR64RegClass, Op0);
2577 }
2578 return Register();
2579}
2580
2581Register fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, Register Op0) {
2582 switch (VT.SimpleTy) {
2583 case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0);
2584 case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0);
2585 default: return Register();
2586 }
2587}
2588
2589// FastEmit functions for ISD::FABS.
2590
2591Register fastEmit_ISD_FABS_MVT_bf16_r(MVT RetVT, Register Op0) {
2592 if (RetVT.SimpleTy != MVT::bf16)
2593 return Register();
2594 if ((Subtarget->hasFullFP16())) {
2595 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSHr, RC: &AArch64::FPR16RegClass, Op0);
2596 }
2597 return Register();
2598}
2599
2600Register fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, Register Op0) {
2601 if (RetVT.SimpleTy != MVT::f16)
2602 return Register();
2603 if ((Subtarget->hasFullFP16())) {
2604 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSHr, RC: &AArch64::FPR16RegClass, Op0);
2605 }
2606 return Register();
2607}
2608
2609Register fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, Register Op0) {
2610 if (RetVT.SimpleTy != MVT::f32)
2611 return Register();
2612 if ((Subtarget->hasFPARMv8())) {
2613 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSSr, RC: &AArch64::FPR32RegClass, Op0);
2614 }
2615 return Register();
2616}
2617
2618Register fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, Register Op0) {
2619 if (RetVT.SimpleTy != MVT::f64)
2620 return Register();
2621 if ((Subtarget->hasFPARMv8())) {
2622 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSDr, RC: &AArch64::FPR64RegClass, Op0);
2623 }
2624 return Register();
2625}
2626
2627Register fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, Register Op0) {
2628 if (RetVT.SimpleTy != MVT::v4f16)
2629 return Register();
2630 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2631 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f16, RC: &AArch64::FPR64RegClass, Op0);
2632 }
2633 return Register();
2634}
2635
2636Register fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, Register Op0) {
2637 if (RetVT.SimpleTy != MVT::v8f16)
2638 return Register();
2639 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2640 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv8f16, RC: &AArch64::FPR128RegClass, Op0);
2641 }
2642 return Register();
2643}
2644
2645Register fastEmit_ISD_FABS_MVT_v4bf16_r(MVT RetVT, Register Op0) {
2646 if (RetVT.SimpleTy != MVT::v4bf16)
2647 return Register();
2648 if ((Subtarget->hasFullFP16())) {
2649 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f16, RC: &AArch64::FPR64RegClass, Op0);
2650 }
2651 return Register();
2652}
2653
2654Register fastEmit_ISD_FABS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
2655 if (RetVT.SimpleTy != MVT::v8bf16)
2656 return Register();
2657 if ((Subtarget->hasFullFP16())) {
2658 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv8f16, RC: &AArch64::FPR128RegClass, Op0);
2659 }
2660 return Register();
2661}
2662
2663Register fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, Register Op0) {
2664 if (RetVT.SimpleTy != MVT::v2f32)
2665 return Register();
2666 if ((Subtarget->isNeonAvailable())) {
2667 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv2f32, RC: &AArch64::FPR64RegClass, Op0);
2668 }
2669 return Register();
2670}
2671
2672Register fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, Register Op0) {
2673 if (RetVT.SimpleTy != MVT::v4f32)
2674 return Register();
2675 if ((Subtarget->isNeonAvailable())) {
2676 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f32, RC: &AArch64::FPR128RegClass, Op0);
2677 }
2678 return Register();
2679}
2680
2681Register fastEmit_ISD_FABS_MVT_v2f64_r(MVT RetVT, Register Op0) {
2682 if (RetVT.SimpleTy != MVT::v2f64)
2683 return Register();
2684 if ((Subtarget->isNeonAvailable())) {
2685 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv2f64, RC: &AArch64::FPR128RegClass, Op0);
2686 }
2687 return Register();
2688}
2689
2690Register fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, Register Op0) {
2691 switch (VT.SimpleTy) {
2692 case MVT::bf16: return fastEmit_ISD_FABS_MVT_bf16_r(RetVT, Op0);
2693 case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0);
2694 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
2695 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
2696 case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0);
2697 case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0);
2698 case MVT::v4bf16: return fastEmit_ISD_FABS_MVT_v4bf16_r(RetVT, Op0);
2699 case MVT::v8bf16: return fastEmit_ISD_FABS_MVT_v8bf16_r(RetVT, Op0);
2700 case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0);
2701 case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0);
2702 case MVT::v2f64: return fastEmit_ISD_FABS_MVT_v2f64_r(RetVT, Op0);
2703 default: return Register();
2704 }
2705}
2706
2707// FastEmit functions for ISD::FCEIL.
2708
2709Register fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
2710 if (RetVT.SimpleTy != MVT::f16)
2711 return Register();
2712 if ((Subtarget->hasFullFP16())) {
2713 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPHr, RC: &AArch64::FPR16RegClass, Op0);
2714 }
2715 return Register();
2716}
2717
2718Register fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
2719 if (RetVT.SimpleTy != MVT::f32)
2720 return Register();
2721 if ((Subtarget->hasFPARMv8())) {
2722 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPSr, RC: &AArch64::FPR32RegClass, Op0);
2723 }
2724 return Register();
2725}
2726
2727Register fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
2728 if (RetVT.SimpleTy != MVT::f64)
2729 return Register();
2730 if ((Subtarget->hasFPARMv8())) {
2731 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPDr, RC: &AArch64::FPR64RegClass, Op0);
2732 }
2733 return Register();
2734}
2735
2736Register fastEmit_ISD_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
2737 if (RetVT.SimpleTy != MVT::v4f16)
2738 return Register();
2739 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2740 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f16, RC: &AArch64::FPR64RegClass, Op0);
2741 }
2742 return Register();
2743}
2744
2745Register fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
2746 if (RetVT.SimpleTy != MVT::v8f16)
2747 return Register();
2748 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2749 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv8f16, RC: &AArch64::FPR128RegClass, Op0);
2750 }
2751 return Register();
2752}
2753
2754Register fastEmit_ISD_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
2755 if (RetVT.SimpleTy != MVT::v2f32)
2756 return Register();
2757 if ((Subtarget->isNeonAvailable())) {
2758 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f32, RC: &AArch64::FPR64RegClass, Op0);
2759 }
2760 return Register();
2761}
2762
2763Register fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
2764 if (RetVT.SimpleTy != MVT::v4f32)
2765 return Register();
2766 if ((Subtarget->isNeonAvailable())) {
2767 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f32, RC: &AArch64::FPR128RegClass, Op0);
2768 }
2769 return Register();
2770}
2771
2772Register fastEmit_ISD_FCEIL_MVT_v2f64_r(MVT RetVT, Register Op0) {
2773 if (RetVT.SimpleTy != MVT::v2f64)
2774 return Register();
2775 if ((Subtarget->isNeonAvailable())) {
2776 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f64, RC: &AArch64::FPR128RegClass, Op0);
2777 }
2778 return Register();
2779}
2780
2781Register fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
2782 switch (VT.SimpleTy) {
2783 case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0);
2784 case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0);
2785 case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0);
2786 case MVT::v4f16: return fastEmit_ISD_FCEIL_MVT_v4f16_r(RetVT, Op0);
2787 case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0);
2788 case MVT::v2f32: return fastEmit_ISD_FCEIL_MVT_v2f32_r(RetVT, Op0);
2789 case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0);
2790 case MVT::v2f64: return fastEmit_ISD_FCEIL_MVT_v2f64_r(RetVT, Op0);
2791 default: return Register();
2792 }
2793}
2794
2795// FastEmit functions for ISD::FFLOOR.
2796
2797Register fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
2798 if (RetVT.SimpleTy != MVT::f16)
2799 return Register();
2800 if ((Subtarget->hasFullFP16())) {
2801 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMHr, RC: &AArch64::FPR16RegClass, Op0);
2802 }
2803 return Register();
2804}
2805
2806Register fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
2807 if (RetVT.SimpleTy != MVT::f32)
2808 return Register();
2809 if ((Subtarget->hasFPARMv8())) {
2810 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMSr, RC: &AArch64::FPR32RegClass, Op0);
2811 }
2812 return Register();
2813}
2814
2815Register fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
2816 if (RetVT.SimpleTy != MVT::f64)
2817 return Register();
2818 if ((Subtarget->hasFPARMv8())) {
2819 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMDr, RC: &AArch64::FPR64RegClass, Op0);
2820 }
2821 return Register();
2822}
2823
2824Register fastEmit_ISD_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
2825 if (RetVT.SimpleTy != MVT::v4f16)
2826 return Register();
2827 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2828 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f16, RC: &AArch64::FPR64RegClass, Op0);
2829 }
2830 return Register();
2831}
2832
2833Register fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
2834 if (RetVT.SimpleTy != MVT::v8f16)
2835 return Register();
2836 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2837 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv8f16, RC: &AArch64::FPR128RegClass, Op0);
2838 }
2839 return Register();
2840}
2841
2842Register fastEmit_ISD_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
2843 if (RetVT.SimpleTy != MVT::v2f32)
2844 return Register();
2845 if ((Subtarget->isNeonAvailable())) {
2846 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f32, RC: &AArch64::FPR64RegClass, Op0);
2847 }
2848 return Register();
2849}
2850
2851Register fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
2852 if (RetVT.SimpleTy != MVT::v4f32)
2853 return Register();
2854 if ((Subtarget->isNeonAvailable())) {
2855 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f32, RC: &AArch64::FPR128RegClass, Op0);
2856 }
2857 return Register();
2858}
2859
2860Register fastEmit_ISD_FFLOOR_MVT_v2f64_r(MVT RetVT, Register Op0) {
2861 if (RetVT.SimpleTy != MVT::v2f64)
2862 return Register();
2863 if ((Subtarget->isNeonAvailable())) {
2864 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f64, RC: &AArch64::FPR128RegClass, Op0);
2865 }
2866 return Register();
2867}
2868
2869Register fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
2870 switch (VT.SimpleTy) {
2871 case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0);
2872 case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0);
2873 case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0);
2874 case MVT::v4f16: return fastEmit_ISD_FFLOOR_MVT_v4f16_r(RetVT, Op0);
2875 case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0);
2876 case MVT::v2f32: return fastEmit_ISD_FFLOOR_MVT_v2f32_r(RetVT, Op0);
2877 case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0);
2878 case MVT::v2f64: return fastEmit_ISD_FFLOOR_MVT_v2f64_r(RetVT, Op0);
2879 default: return Register();
2880 }
2881}
2882
2883// FastEmit functions for ISD::FNEARBYINT.
2884
2885Register fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
2886 if (RetVT.SimpleTy != MVT::f16)
2887 return Register();
2888 if ((Subtarget->hasFullFP16())) {
2889 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIHr, RC: &AArch64::FPR16RegClass, Op0);
2890 }
2891 return Register();
2892}
2893
2894Register fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
2895 if (RetVT.SimpleTy != MVT::f32)
2896 return Register();
2897 if ((Subtarget->hasFPARMv8())) {
2898 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTISr, RC: &AArch64::FPR32RegClass, Op0);
2899 }
2900 return Register();
2901}
2902
2903Register fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
2904 if (RetVT.SimpleTy != MVT::f64)
2905 return Register();
2906 if ((Subtarget->hasFPARMv8())) {
2907 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIDr, RC: &AArch64::FPR64RegClass, Op0);
2908 }
2909 return Register();
2910}
2911
2912Register fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2913 if (RetVT.SimpleTy != MVT::v4f16)
2914 return Register();
2915 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2916 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f16, RC: &AArch64::FPR64RegClass, Op0);
2917 }
2918 return Register();
2919}
2920
2921Register fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2922 if (RetVT.SimpleTy != MVT::v8f16)
2923 return Register();
2924 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2925 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv8f16, RC: &AArch64::FPR128RegClass, Op0);
2926 }
2927 return Register();
2928}
2929
2930Register fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2931 if (RetVT.SimpleTy != MVT::v2f32)
2932 return Register();
2933 if ((Subtarget->isNeonAvailable())) {
2934 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f32, RC: &AArch64::FPR64RegClass, Op0);
2935 }
2936 return Register();
2937}
2938
2939Register fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2940 if (RetVT.SimpleTy != MVT::v4f32)
2941 return Register();
2942 if ((Subtarget->isNeonAvailable())) {
2943 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f32, RC: &AArch64::FPR128RegClass, Op0);
2944 }
2945 return Register();
2946}
2947
2948Register fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
2949 if (RetVT.SimpleTy != MVT::v2f64)
2950 return Register();
2951 if ((Subtarget->isNeonAvailable())) {
2952 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f64, RC: &AArch64::FPR128RegClass, Op0);
2953 }
2954 return Register();
2955}
2956
2957Register fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
2958 switch (VT.SimpleTy) {
2959 case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0);
2960 case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0);
2961 case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0);
2962 case MVT::v4f16: return fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(RetVT, Op0);
2963 case MVT::v8f16: return fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(RetVT, Op0);
2964 case MVT::v2f32: return fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(RetVT, Op0);
2965 case MVT::v4f32: return fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(RetVT, Op0);
2966 case MVT::v2f64: return fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(RetVT, Op0);
2967 default: return Register();
2968 }
2969}
2970
2971// FastEmit functions for ISD::FNEG.
2972
2973Register fastEmit_ISD_FNEG_MVT_bf16_r(MVT RetVT, Register Op0) {
2974 if (RetVT.SimpleTy != MVT::bf16)
2975 return Register();
2976 if ((Subtarget->hasFullFP16())) {
2977 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGHr, RC: &AArch64::FPR16RegClass, Op0);
2978 }
2979 return Register();
2980}
2981
2982Register fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, Register Op0) {
2983 if (RetVT.SimpleTy != MVT::f16)
2984 return Register();
2985 if ((Subtarget->hasFullFP16())) {
2986 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGHr, RC: &AArch64::FPR16RegClass, Op0);
2987 }
2988 return Register();
2989}
2990
2991Register fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, Register Op0) {
2992 if (RetVT.SimpleTy != MVT::f32)
2993 return Register();
2994 if ((Subtarget->hasFPARMv8())) {
2995 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGSr, RC: &AArch64::FPR32RegClass, Op0);
2996 }
2997 return Register();
2998}
2999
3000Register fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, Register Op0) {
3001 if (RetVT.SimpleTy != MVT::f64)
3002 return Register();
3003 if ((Subtarget->hasFPARMv8())) {
3004 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGDr, RC: &AArch64::FPR64RegClass, Op0);
3005 }
3006 return Register();
3007}
3008
3009Register fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, Register Op0) {
3010 if (RetVT.SimpleTy != MVT::v4f16)
3011 return Register();
3012 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3013 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f16, RC: &AArch64::FPR64RegClass, Op0);
3014 }
3015 return Register();
3016}
3017
3018Register fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, Register Op0) {
3019 if (RetVT.SimpleTy != MVT::v8f16)
3020 return Register();
3021 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3022 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv8f16, RC: &AArch64::FPR128RegClass, Op0);
3023 }
3024 return Register();
3025}
3026
3027Register fastEmit_ISD_FNEG_MVT_v4bf16_r(MVT RetVT, Register Op0) {
3028 if (RetVT.SimpleTy != MVT::v4bf16)
3029 return Register();
3030 if ((Subtarget->hasFullFP16())) {
3031 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f16, RC: &AArch64::FPR64RegClass, Op0);
3032 }
3033 return Register();
3034}
3035
3036Register fastEmit_ISD_FNEG_MVT_v8bf16_r(MVT RetVT, Register Op0) {
3037 if (RetVT.SimpleTy != MVT::v8bf16)
3038 return Register();
3039 if ((Subtarget->hasFullFP16())) {
3040 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv8f16, RC: &AArch64::FPR128RegClass, Op0);
3041 }
3042 return Register();
3043}
3044
3045Register fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, Register Op0) {
3046 if (RetVT.SimpleTy != MVT::v2f32)
3047 return Register();
3048 if ((Subtarget->isNeonAvailable())) {
3049 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv2f32, RC: &AArch64::FPR64RegClass, Op0);
3050 }
3051 return Register();
3052}
3053
3054Register fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, Register Op0) {
3055 if (RetVT.SimpleTy != MVT::v4f32)
3056 return Register();
3057 if ((Subtarget->isNeonAvailable())) {
3058 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f32, RC: &AArch64::FPR128RegClass, Op0);
3059 }
3060 return Register();
3061}
3062
3063Register fastEmit_ISD_FNEG_MVT_v2f64_r(MVT RetVT, Register Op0) {
3064 if (RetVT.SimpleTy != MVT::v2f64)
3065 return Register();
3066 if ((Subtarget->isNeonAvailable())) {
3067 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv2f64, RC: &AArch64::FPR128RegClass, Op0);
3068 }
3069 return Register();
3070}
3071
3072Register fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, Register Op0) {
3073 switch (VT.SimpleTy) {
3074 case MVT::bf16: return fastEmit_ISD_FNEG_MVT_bf16_r(RetVT, Op0);
3075 case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0);
3076 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
3077 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
3078 case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0);
3079 case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0);
3080 case MVT::v4bf16: return fastEmit_ISD_FNEG_MVT_v4bf16_r(RetVT, Op0);
3081 case MVT::v8bf16: return fastEmit_ISD_FNEG_MVT_v8bf16_r(RetVT, Op0);
3082 case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0);
3083 case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0);
3084 case MVT::v2f64: return fastEmit_ISD_FNEG_MVT_v2f64_r(RetVT, Op0);
3085 default: return Register();
3086 }
3087}
3088
3089// FastEmit functions for ISD::FP_EXTEND.
3090
3091Register fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Register Op0) {
3092 if ((Subtarget->hasFPARMv8())) {
3093 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSHr, RC: &AArch64::FPR32RegClass, Op0);
3094 }
3095 return Register();
3096}
3097
3098Register fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Register Op0) {
3099 if ((Subtarget->hasFPARMv8())) {
3100 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDHr, RC: &AArch64::FPR64RegClass, Op0);
3101 }
3102 return Register();
3103}
3104
3105Register fastEmit_ISD_FP_EXTEND_MVT_f16_r(MVT RetVT, Register Op0) {
3106switch (RetVT.SimpleTy) {
3107 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Op0);
3108 case MVT::f64: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Op0);
3109 default: return Register();
3110}
3111}
3112
3113Register fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
3114 if (RetVT.SimpleTy != MVT::f64)
3115 return Register();
3116 if ((Subtarget->hasFPARMv8())) {
3117 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDSr, RC: &AArch64::FPR64RegClass, Op0);
3118 }
3119 return Register();
3120}
3121
3122Register fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3123 if (RetVT.SimpleTy != MVT::v4f32)
3124 return Register();
3125 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv4i16, RC: &AArch64::FPR128RegClass, Op0);
3126}
3127
3128Register fastEmit_ISD_FP_EXTEND_MVT_v4bf16_r(MVT RetVT, Register Op0) {
3129 if (RetVT.SimpleTy != MVT::v4f32)
3130 return Register();
3131 if ((Subtarget->isNeonAvailable())) {
3132 return fastEmitInst_r(MachineInstOpcode: AArch64::SHLLv4i16, RC: &AArch64::FPR128RegClass, Op0);
3133 }
3134 return Register();
3135}
3136
3137Register fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3138 if (RetVT.SimpleTy != MVT::v2f64)
3139 return Register();
3140 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv2i32, RC: &AArch64::FPR128RegClass, Op0);
3141}
3142
3143Register fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3144 switch (VT.SimpleTy) {
3145 case MVT::f16: return fastEmit_ISD_FP_EXTEND_MVT_f16_r(RetVT, Op0);
3146 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
3147 case MVT::v4f16: return fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
3148 case MVT::v4bf16: return fastEmit_ISD_FP_EXTEND_MVT_v4bf16_r(RetVT, Op0);
3149 case MVT::v2f32: return fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(RetVT, Op0);
3150 default: return Register();
3151 }
3152}
3153
3154// FastEmit functions for ISD::FP_ROUND.
3155
3156Register fastEmit_ISD_FP_ROUND_MVT_f32_MVT_bf16_r(Register Op0) {
3157 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
3158 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVT, RC: &AArch64::FPR16RegClass, Op0);
3159 }
3160 return Register();
3161}
3162
3163Register fastEmit_ISD_FP_ROUND_MVT_f32_MVT_f16_r(Register Op0) {
3164 if ((Subtarget->hasFPARMv8())) {
3165 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHSr, RC: &AArch64::FPR16RegClass, Op0);
3166 }
3167 return Register();
3168}
3169
3170Register fastEmit_ISD_FP_ROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3171switch (RetVT.SimpleTy) {
3172 case MVT::bf16: return fastEmit_ISD_FP_ROUND_MVT_f32_MVT_bf16_r(Op0);
3173 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f32_MVT_f16_r(Op0);
3174 default: return Register();
3175}
3176}
3177
3178Register fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Register Op0) {
3179 if ((Subtarget->hasFPARMv8())) {
3180 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHDr, RC: &AArch64::FPR16RegClass, Op0);
3181 }
3182 return Register();
3183}
3184
3185Register fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Register Op0) {
3186 if ((Subtarget->hasFPARMv8())) {
3187 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSDr, RC: &AArch64::FPR32RegClass, Op0);
3188 }
3189 return Register();
3190}
3191
3192Register fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3193switch (RetVT.SimpleTy) {
3194 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Op0);
3195 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Op0);
3196 default: return Register();
3197}
3198}
3199
3200Register fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Register Op0) {
3201 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
3202}
3203
3204Register fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Register Op0) {
3205 if ((Subtarget->hasBF16()) && (Subtarget->isNeonAvailable())) {
3206 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVTN, RC: &AArch64::FPR64RegClass, Op0);
3207 }
3208 return Register();
3209}
3210
3211Register fastEmit_ISD_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3212switch (RetVT.SimpleTy) {
3213 case MVT::v4f16: return fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Op0);
3214 case MVT::v4bf16: return fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Op0);
3215 default: return Register();
3216}
3217}
3218
3219Register fastEmit_ISD_FP_ROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
3220 if (RetVT.SimpleTy != MVT::v2f32)
3221 return Register();
3222 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
3223}
3224
3225Register fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
3226 switch (VT.SimpleTy) {
3227 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f32_r(RetVT, Op0);
3228 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
3229 case MVT::v4f32: return fastEmit_ISD_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
3230 case MVT::v2f64: return fastEmit_ISD_FP_ROUND_MVT_v2f64_r(RetVT, Op0);
3231 default: return Register();
3232 }
3233}
3234
3235// FastEmit functions for ISD::FP_TO_SINT.
3236
3237Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
3238 if ((Subtarget->hasFPRCVT())) {
3239 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSHr, RC: &AArch64::FPR32RegClass, Op0);
3240 }
3241 if ((Subtarget->hasFullFP16())) {
3242 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
3243 }
3244 return Register();
3245}
3246
3247Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
3248 if ((Subtarget->hasFPRCVT())) {
3249 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDHr, RC: &AArch64::FPR64RegClass, Op0);
3250 }
3251 if ((Subtarget->hasFullFP16())) {
3252 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
3253 }
3254 return Register();
3255}
3256
3257Register fastEmit_ISD_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
3258switch (RetVT.SimpleTy) {
3259 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
3260 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
3261 default: return Register();
3262}
3263}
3264
3265Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
3266 if ((Subtarget->hasFPARMv8())) {
3267 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
3268 }
3269 return Register();
3270}
3271
3272Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
3273 if ((Subtarget->hasFPRCVT())) {
3274 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDSr, RC: &AArch64::FPR64RegClass, Op0);
3275 }
3276 if ((Subtarget->hasFPARMv8())) {
3277 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
3278 }
3279 return Register();
3280}
3281
3282Register fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
3283switch (RetVT.SimpleTy) {
3284 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
3285 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
3286 default: return Register();
3287}
3288}
3289
3290Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
3291 if ((Subtarget->hasFPRCVT())) {
3292 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSDr, RC: &AArch64::FPR32RegClass, Op0);
3293 }
3294 if ((Subtarget->hasFPARMv8())) {
3295 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
3296 }
3297 return Register();
3298}
3299
3300Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
3301 if ((Subtarget->hasFPARMv8())) {
3302 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
3303 }
3304 return Register();
3305}
3306
3307Register fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
3308switch (RetVT.SimpleTy) {
3309 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
3310 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
3311 default: return Register();
3312}
3313}
3314
3315Register fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3316 if (RetVT.SimpleTy != MVT::v4i16)
3317 return Register();
3318 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3319 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
3320 }
3321 return Register();
3322}
3323
3324Register fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3325 if (RetVT.SimpleTy != MVT::v8i16)
3326 return Register();
3327 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3328 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
3329 }
3330 return Register();
3331}
3332
3333Register fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3334 if (RetVT.SimpleTy != MVT::v2i32)
3335 return Register();
3336 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3337 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
3338 }
3339 return Register();
3340}
3341
3342Register fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3343 if (RetVT.SimpleTy != MVT::v4i32)
3344 return Register();
3345 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3346 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
3347 }
3348 return Register();
3349}
3350
3351Register fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3352 if (RetVT.SimpleTy != MVT::v2i64)
3353 return Register();
3354 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3355 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
3356 }
3357 return Register();
3358}
3359
3360Register fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
3361 switch (VT.SimpleTy) {
3362 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
3363 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
3364 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
3365 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
3366 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
3367 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
3368 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
3369 case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0);
3370 default: return Register();
3371 }
3372}
3373
3374// FastEmit functions for ISD::FP_TO_UINT.
3375
3376Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
3377 if ((Subtarget->hasFPRCVT())) {
3378 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSHr, RC: &AArch64::FPR32RegClass, Op0);
3379 }
3380 if ((Subtarget->hasFullFP16())) {
3381 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
3382 }
3383 return Register();
3384}
3385
3386Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
3387 if ((Subtarget->hasFPRCVT())) {
3388 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDHr, RC: &AArch64::FPR64RegClass, Op0);
3389 }
3390 if ((Subtarget->hasFullFP16())) {
3391 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
3392 }
3393 return Register();
3394}
3395
3396Register fastEmit_ISD_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
3397switch (RetVT.SimpleTy) {
3398 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
3399 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
3400 default: return Register();
3401}
3402}
3403
3404Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
3405 if ((Subtarget->hasFPARMv8())) {
3406 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
3407 }
3408 return Register();
3409}
3410
3411Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
3412 if ((Subtarget->hasFPRCVT())) {
3413 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDSr, RC: &AArch64::FPR64RegClass, Op0);
3414 }
3415 if ((Subtarget->hasFPARMv8())) {
3416 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
3417 }
3418 return Register();
3419}
3420
3421Register fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
3422switch (RetVT.SimpleTy) {
3423 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
3424 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
3425 default: return Register();
3426}
3427}
3428
3429Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
3430 if ((Subtarget->hasFPRCVT())) {
3431 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSDr, RC: &AArch64::FPR32RegClass, Op0);
3432 }
3433 if ((Subtarget->hasFPARMv8())) {
3434 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
3435 }
3436 return Register();
3437}
3438
3439Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
3440 if ((Subtarget->hasFPARMv8())) {
3441 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
3442 }
3443 return Register();
3444}
3445
3446Register fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
3447switch (RetVT.SimpleTy) {
3448 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
3449 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
3450 default: return Register();
3451}
3452}
3453
3454Register fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3455 if (RetVT.SimpleTy != MVT::v4i16)
3456 return Register();
3457 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3458 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
3459 }
3460 return Register();
3461}
3462
3463Register fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3464 if (RetVT.SimpleTy != MVT::v8i16)
3465 return Register();
3466 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3467 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
3468 }
3469 return Register();
3470}
3471
3472Register fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3473 if (RetVT.SimpleTy != MVT::v2i32)
3474 return Register();
3475 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3476 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
3477 }
3478 return Register();
3479}
3480
3481Register fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3482 if (RetVT.SimpleTy != MVT::v4i32)
3483 return Register();
3484 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3485 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
3486 }
3487 return Register();
3488}
3489
3490Register fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3491 if (RetVT.SimpleTy != MVT::v2i64)
3492 return Register();
3493 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3494 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
3495 }
3496 return Register();
3497}
3498
3499Register fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
3500 switch (VT.SimpleTy) {
3501 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
3502 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
3503 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
3504 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
3505 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
3506 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
3507 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
3508 case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0);
3509 default: return Register();
3510 }
3511}
3512
3513// FastEmit functions for ISD::FRINT.
3514
3515Register fastEmit_ISD_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
3516 if (RetVT.SimpleTy != MVT::f16)
3517 return Register();
3518 if ((Subtarget->hasFullFP16())) {
3519 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXHr, RC: &AArch64::FPR16RegClass, Op0);
3520 }
3521 return Register();
3522}
3523
3524Register fastEmit_ISD_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
3525 if (RetVT.SimpleTy != MVT::f32)
3526 return Register();
3527 if ((Subtarget->hasFPARMv8())) {
3528 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXSr, RC: &AArch64::FPR32RegClass, Op0);
3529 }
3530 return Register();
3531}
3532
3533Register fastEmit_ISD_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
3534 if (RetVT.SimpleTy != MVT::f64)
3535 return Register();
3536 if ((Subtarget->hasFPARMv8())) {
3537 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXDr, RC: &AArch64::FPR64RegClass, Op0);
3538 }
3539 return Register();
3540}
3541
3542Register fastEmit_ISD_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3543 if (RetVT.SimpleTy != MVT::v4f16)
3544 return Register();
3545 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3546 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f16, RC: &AArch64::FPR64RegClass, Op0);
3547 }
3548 return Register();
3549}
3550
3551Register fastEmit_ISD_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3552 if (RetVT.SimpleTy != MVT::v8f16)
3553 return Register();
3554 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3555 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv8f16, RC: &AArch64::FPR128RegClass, Op0);
3556 }
3557 return Register();
3558}
3559
3560Register fastEmit_ISD_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3561 if (RetVT.SimpleTy != MVT::v2f32)
3562 return Register();
3563 if ((Subtarget->isNeonAvailable())) {
3564 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f32, RC: &AArch64::FPR64RegClass, Op0);
3565 }
3566 return Register();
3567}
3568
3569Register fastEmit_ISD_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3570 if (RetVT.SimpleTy != MVT::v4f32)
3571 return Register();
3572 if ((Subtarget->isNeonAvailable())) {
3573 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f32, RC: &AArch64::FPR128RegClass, Op0);
3574 }
3575 return Register();
3576}
3577
3578Register fastEmit_ISD_FRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3579 if (RetVT.SimpleTy != MVT::v2f64)
3580 return Register();
3581 if ((Subtarget->isNeonAvailable())) {
3582 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f64, RC: &AArch64::FPR128RegClass, Op0);
3583 }
3584 return Register();
3585}
3586
3587Register fastEmit_ISD_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
3588 switch (VT.SimpleTy) {
3589 case MVT::f16: return fastEmit_ISD_FRINT_MVT_f16_r(RetVT, Op0);
3590 case MVT::f32: return fastEmit_ISD_FRINT_MVT_f32_r(RetVT, Op0);
3591 case MVT::f64: return fastEmit_ISD_FRINT_MVT_f64_r(RetVT, Op0);
3592 case MVT::v4f16: return fastEmit_ISD_FRINT_MVT_v4f16_r(RetVT, Op0);
3593 case MVT::v8f16: return fastEmit_ISD_FRINT_MVT_v8f16_r(RetVT, Op0);
3594 case MVT::v2f32: return fastEmit_ISD_FRINT_MVT_v2f32_r(RetVT, Op0);
3595 case MVT::v4f32: return fastEmit_ISD_FRINT_MVT_v4f32_r(RetVT, Op0);
3596 case MVT::v2f64: return fastEmit_ISD_FRINT_MVT_v2f64_r(RetVT, Op0);
3597 default: return Register();
3598 }
3599}
3600
3601// FastEmit functions for ISD::FROUND.
3602
3603Register fastEmit_ISD_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
3604 if (RetVT.SimpleTy != MVT::f16)
3605 return Register();
3606 if ((Subtarget->hasFullFP16())) {
3607 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAHr, RC: &AArch64::FPR16RegClass, Op0);
3608 }
3609 return Register();
3610}
3611
3612Register fastEmit_ISD_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3613 if (RetVT.SimpleTy != MVT::f32)
3614 return Register();
3615 if ((Subtarget->hasFPARMv8())) {
3616 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTASr, RC: &AArch64::FPR32RegClass, Op0);
3617 }
3618 return Register();
3619}
3620
3621Register fastEmit_ISD_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3622 if (RetVT.SimpleTy != MVT::f64)
3623 return Register();
3624 if ((Subtarget->hasFPARMv8())) {
3625 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTADr, RC: &AArch64::FPR64RegClass, Op0);
3626 }
3627 return Register();
3628}
3629
3630Register fastEmit_ISD_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3631 if (RetVT.SimpleTy != MVT::v4f16)
3632 return Register();
3633 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3634 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f16, RC: &AArch64::FPR64RegClass, Op0);
3635 }
3636 return Register();
3637}
3638
3639Register fastEmit_ISD_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
3640 if (RetVT.SimpleTy != MVT::v8f16)
3641 return Register();
3642 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3643 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv8f16, RC: &AArch64::FPR128RegClass, Op0);
3644 }
3645 return Register();
3646}
3647
3648Register fastEmit_ISD_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3649 if (RetVT.SimpleTy != MVT::v2f32)
3650 return Register();
3651 if ((Subtarget->isNeonAvailable())) {
3652 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f32, RC: &AArch64::FPR64RegClass, Op0);
3653 }
3654 return Register();
3655}
3656
3657Register fastEmit_ISD_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3658 if (RetVT.SimpleTy != MVT::v4f32)
3659 return Register();
3660 if ((Subtarget->isNeonAvailable())) {
3661 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f32, RC: &AArch64::FPR128RegClass, Op0);
3662 }
3663 return Register();
3664}
3665
3666Register fastEmit_ISD_FROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
3667 if (RetVT.SimpleTy != MVT::v2f64)
3668 return Register();
3669 if ((Subtarget->isNeonAvailable())) {
3670 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f64, RC: &AArch64::FPR128RegClass, Op0);
3671 }
3672 return Register();
3673}
3674
3675Register fastEmit_ISD_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
3676 switch (VT.SimpleTy) {
3677 case MVT::f16: return fastEmit_ISD_FROUND_MVT_f16_r(RetVT, Op0);
3678 case MVT::f32: return fastEmit_ISD_FROUND_MVT_f32_r(RetVT, Op0);
3679 case MVT::f64: return fastEmit_ISD_FROUND_MVT_f64_r(RetVT, Op0);
3680 case MVT::v4f16: return fastEmit_ISD_FROUND_MVT_v4f16_r(RetVT, Op0);
3681 case MVT::v8f16: return fastEmit_ISD_FROUND_MVT_v8f16_r(RetVT, Op0);
3682 case MVT::v2f32: return fastEmit_ISD_FROUND_MVT_v2f32_r(RetVT, Op0);
3683 case MVT::v4f32: return fastEmit_ISD_FROUND_MVT_v4f32_r(RetVT, Op0);
3684 case MVT::v2f64: return fastEmit_ISD_FROUND_MVT_v2f64_r(RetVT, Op0);
3685 default: return Register();
3686 }
3687}
3688
3689// FastEmit functions for ISD::FROUNDEVEN.
3690
3691Register fastEmit_ISD_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
3692 if (RetVT.SimpleTy != MVT::f16)
3693 return Register();
3694 if ((Subtarget->hasFullFP16())) {
3695 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNHr, RC: &AArch64::FPR16RegClass, Op0);
3696 }
3697 return Register();
3698}
3699
3700Register fastEmit_ISD_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
3701 if (RetVT.SimpleTy != MVT::f32)
3702 return Register();
3703 if ((Subtarget->hasFPARMv8())) {
3704 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNSr, RC: &AArch64::FPR32RegClass, Op0);
3705 }
3706 return Register();
3707}
3708
3709Register fastEmit_ISD_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
3710 if (RetVT.SimpleTy != MVT::f64)
3711 return Register();
3712 if ((Subtarget->hasFPARMv8())) {
3713 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNDr, RC: &AArch64::FPR64RegClass, Op0);
3714 }
3715 return Register();
3716}
3717
3718Register fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
3719 if (RetVT.SimpleTy != MVT::v4f16)
3720 return Register();
3721 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3722 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f16, RC: &AArch64::FPR64RegClass, Op0);
3723 }
3724 return Register();
3725}
3726
3727Register fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
3728 if (RetVT.SimpleTy != MVT::v8f16)
3729 return Register();
3730 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3731 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv8f16, RC: &AArch64::FPR128RegClass, Op0);
3732 }
3733 return Register();
3734}
3735
3736Register fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
3737 if (RetVT.SimpleTy != MVT::v2f32)
3738 return Register();
3739 if ((Subtarget->isNeonAvailable())) {
3740 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f32, RC: &AArch64::FPR64RegClass, Op0);
3741 }
3742 return Register();
3743}
3744
3745Register fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
3746 if (RetVT.SimpleTy != MVT::v4f32)
3747 return Register();
3748 if ((Subtarget->isNeonAvailable())) {
3749 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f32, RC: &AArch64::FPR128RegClass, Op0);
3750 }
3751 return Register();
3752}
3753
3754Register fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, Register Op0) {
3755 if (RetVT.SimpleTy != MVT::v2f64)
3756 return Register();
3757 if ((Subtarget->isNeonAvailable())) {
3758 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f64, RC: &AArch64::FPR128RegClass, Op0);
3759 }
3760 return Register();
3761}
3762
3763Register fastEmit_ISD_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
3764 switch (VT.SimpleTy) {
3765 case MVT::f16: return fastEmit_ISD_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
3766 case MVT::f32: return fastEmit_ISD_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
3767 case MVT::f64: return fastEmit_ISD_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
3768 case MVT::v4f16: return fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
3769 case MVT::v8f16: return fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
3770 case MVT::v2f32: return fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
3771 case MVT::v4f32: return fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
3772 case MVT::v2f64: return fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0);
3773 default: return Register();
3774 }
3775}
3776
3777// FastEmit functions for ISD::FSQRT.
3778
3779Register fastEmit_ISD_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
3780 if (RetVT.SimpleTy != MVT::f16)
3781 return Register();
3782 if ((Subtarget->hasFullFP16())) {
3783 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTHr, RC: &AArch64::FPR16RegClass, Op0);
3784 }
3785 return Register();
3786}
3787
3788Register fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
3789 if (RetVT.SimpleTy != MVT::f32)
3790 return Register();
3791 if ((Subtarget->hasFPARMv8())) {
3792 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTSr, RC: &AArch64::FPR32RegClass, Op0);
3793 }
3794 return Register();
3795}
3796
3797Register fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
3798 if (RetVT.SimpleTy != MVT::f64)
3799 return Register();
3800 if ((Subtarget->hasFPARMv8())) {
3801 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTDr, RC: &AArch64::FPR64RegClass, Op0);
3802 }
3803 return Register();
3804}
3805
3806Register fastEmit_ISD_FSQRT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3807 if (RetVT.SimpleTy != MVT::v4f16)
3808 return Register();
3809 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3810 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f16, RC: &AArch64::FPR64RegClass, Op0);
3811 }
3812 return Register();
3813}
3814
3815Register fastEmit_ISD_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3816 if (RetVT.SimpleTy != MVT::v8f16)
3817 return Register();
3818 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3819 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv8f16, RC: &AArch64::FPR128RegClass, Op0);
3820 }
3821 return Register();
3822}
3823
3824Register fastEmit_ISD_FSQRT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3825 if (RetVT.SimpleTy != MVT::v2f32)
3826 return Register();
3827 if ((Subtarget->isNeonAvailable())) {
3828 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f32, RC: &AArch64::FPR64RegClass, Op0);
3829 }
3830 return Register();
3831}
3832
3833Register fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3834 if (RetVT.SimpleTy != MVT::v4f32)
3835 return Register();
3836 if ((Subtarget->isNeonAvailable())) {
3837 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f32, RC: &AArch64::FPR128RegClass, Op0);
3838 }
3839 return Register();
3840}
3841
3842Register fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3843 if (RetVT.SimpleTy != MVT::v2f64)
3844 return Register();
3845 if ((Subtarget->isNeonAvailable())) {
3846 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f64, RC: &AArch64::FPR128RegClass, Op0);
3847 }
3848 return Register();
3849}
3850
3851Register fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
3852 switch (VT.SimpleTy) {
3853 case MVT::f16: return fastEmit_ISD_FSQRT_MVT_f16_r(RetVT, Op0);
3854 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
3855 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
3856 case MVT::v4f16: return fastEmit_ISD_FSQRT_MVT_v4f16_r(RetVT, Op0);
3857 case MVT::v8f16: return fastEmit_ISD_FSQRT_MVT_v8f16_r(RetVT, Op0);
3858 case MVT::v2f32: return fastEmit_ISD_FSQRT_MVT_v2f32_r(RetVT, Op0);
3859 case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0);
3860 case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0);
3861 default: return Register();
3862 }
3863}
3864
3865// FastEmit functions for ISD::FTRUNC.
3866
3867Register fastEmit_ISD_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
3868 if (RetVT.SimpleTy != MVT::f16)
3869 return Register();
3870 if ((Subtarget->hasFullFP16())) {
3871 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZHr, RC: &AArch64::FPR16RegClass, Op0);
3872 }
3873 return Register();
3874}
3875
3876Register fastEmit_ISD_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
3877 if (RetVT.SimpleTy != MVT::f32)
3878 return Register();
3879 if ((Subtarget->hasFPARMv8())) {
3880 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZSr, RC: &AArch64::FPR32RegClass, Op0);
3881 }
3882 return Register();
3883}
3884
3885Register fastEmit_ISD_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
3886 if (RetVT.SimpleTy != MVT::f64)
3887 return Register();
3888 if ((Subtarget->hasFPARMv8())) {
3889 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZDr, RC: &AArch64::FPR64RegClass, Op0);
3890 }
3891 return Register();
3892}
3893
3894Register fastEmit_ISD_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
3895 if (RetVT.SimpleTy != MVT::v4f16)
3896 return Register();
3897 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3898 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f16, RC: &AArch64::FPR64RegClass, Op0);
3899 }
3900 return Register();
3901}
3902
3903Register fastEmit_ISD_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
3904 if (RetVT.SimpleTy != MVT::v8f16)
3905 return Register();
3906 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3907 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv8f16, RC: &AArch64::FPR128RegClass, Op0);
3908 }
3909 return Register();
3910}
3911
3912Register fastEmit_ISD_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
3913 if (RetVT.SimpleTy != MVT::v2f32)
3914 return Register();
3915 if ((Subtarget->isNeonAvailable())) {
3916 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f32, RC: &AArch64::FPR64RegClass, Op0);
3917 }
3918 return Register();
3919}
3920
3921Register fastEmit_ISD_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
3922 if (RetVT.SimpleTy != MVT::v4f32)
3923 return Register();
3924 if ((Subtarget->isNeonAvailable())) {
3925 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f32, RC: &AArch64::FPR128RegClass, Op0);
3926 }
3927 return Register();
3928}
3929
3930Register fastEmit_ISD_FTRUNC_MVT_v2f64_r(MVT RetVT, Register Op0) {
3931 if (RetVT.SimpleTy != MVT::v2f64)
3932 return Register();
3933 if ((Subtarget->isNeonAvailable())) {
3934 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f64, RC: &AArch64::FPR128RegClass, Op0);
3935 }
3936 return Register();
3937}
3938
3939Register fastEmit_ISD_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
3940 switch (VT.SimpleTy) {
3941 case MVT::f16: return fastEmit_ISD_FTRUNC_MVT_f16_r(RetVT, Op0);
3942 case MVT::f32: return fastEmit_ISD_FTRUNC_MVT_f32_r(RetVT, Op0);
3943 case MVT::f64: return fastEmit_ISD_FTRUNC_MVT_f64_r(RetVT, Op0);
3944 case MVT::v4f16: return fastEmit_ISD_FTRUNC_MVT_v4f16_r(RetVT, Op0);
3945 case MVT::v8f16: return fastEmit_ISD_FTRUNC_MVT_v8f16_r(RetVT, Op0);
3946 case MVT::v2f32: return fastEmit_ISD_FTRUNC_MVT_v2f32_r(RetVT, Op0);
3947 case MVT::v4f32: return fastEmit_ISD_FTRUNC_MVT_v4f32_r(RetVT, Op0);
3948 case MVT::v2f64: return fastEmit_ISD_FTRUNC_MVT_v2f64_r(RetVT, Op0);
3949 default: return Register();
3950 }
3951}
3952
3953// FastEmit functions for ISD::LLROUND.
3954
3955Register fastEmit_ISD_LLROUND_MVT_f16_r(MVT RetVT, Register Op0) {
3956 if (RetVT.SimpleTy != MVT::i64)
3957 return Register();
3958 if ((Subtarget->hasFPRCVT())) {
3959 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDHr, RC: &AArch64::FPR64RegClass, Op0);
3960 }
3961 if ((Subtarget->hasFullFP16())) {
3962 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
3963 }
3964 return Register();
3965}
3966
3967Register fastEmit_ISD_LLROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3968 if (RetVT.SimpleTy != MVT::i64)
3969 return Register();
3970 if ((Subtarget->hasFPRCVT())) {
3971 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDSr, RC: &AArch64::FPR64RegClass, Op0);
3972 }
3973 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
3974}
3975
3976Register fastEmit_ISD_LLROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3977 if (RetVT.SimpleTy != MVT::i64)
3978 return Register();
3979 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
3980}
3981
3982Register fastEmit_ISD_LLROUND_r(MVT VT, MVT RetVT, Register Op0) {
3983 switch (VT.SimpleTy) {
3984 case MVT::f16: return fastEmit_ISD_LLROUND_MVT_f16_r(RetVT, Op0);
3985 case MVT::f32: return fastEmit_ISD_LLROUND_MVT_f32_r(RetVT, Op0);
3986 case MVT::f64: return fastEmit_ISD_LLROUND_MVT_f64_r(RetVT, Op0);
3987 default: return Register();
3988 }
3989}
3990
3991// FastEmit functions for ISD::LROUND.
3992
3993Register fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(Register Op0) {
3994 if ((Subtarget->hasFPRCVT())) {
3995 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASSHr, RC: &AArch64::FPR32RegClass, Op0);
3996 }
3997 if ((Subtarget->hasFullFP16())) {
3998 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWHr, RC: &AArch64::GPR32RegClass, Op0);
3999 }
4000 return Register();
4001}
4002
4003Register fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(Register Op0) {
4004 if ((Subtarget->hasFPRCVT())) {
4005 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDHr, RC: &AArch64::FPR64RegClass, Op0);
4006 }
4007 if ((Subtarget->hasFullFP16())) {
4008 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
4009 }
4010 return Register();
4011}
4012
4013Register fastEmit_ISD_LROUND_MVT_f16_r(MVT RetVT, Register Op0) {
4014switch (RetVT.SimpleTy) {
4015 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(Op0);
4016 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(Op0);
4017 default: return Register();
4018}
4019}
4020
4021Register fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(Register Op0) {
4022 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWSr, RC: &AArch64::GPR32RegClass, Op0);
4023}
4024
4025Register fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(Register Op0) {
4026 if ((Subtarget->hasFPRCVT())) {
4027 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDSr, RC: &AArch64::FPR64RegClass, Op0);
4028 }
4029 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
4030}
4031
4032Register fastEmit_ISD_LROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4033switch (RetVT.SimpleTy) {
4034 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(Op0);
4035 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(Op0);
4036 default: return Register();
4037}
4038}
4039
4040Register fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(Register Op0) {
4041 if ((Subtarget->hasFPRCVT())) {
4042 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASSDr, RC: &AArch64::FPR32RegClass, Op0);
4043 }
4044 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWDr, RC: &AArch64::GPR32RegClass, Op0);
4045}
4046
4047Register fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(Register Op0) {
4048 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
4049}
4050
4051Register fastEmit_ISD_LROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4052switch (RetVT.SimpleTy) {
4053 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(Op0);
4054 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(Op0);
4055 default: return Register();
4056}
4057}
4058
4059Register fastEmit_ISD_LROUND_r(MVT VT, MVT RetVT, Register Op0) {
4060 switch (VT.SimpleTy) {
4061 case MVT::f16: return fastEmit_ISD_LROUND_MVT_f16_r(RetVT, Op0);
4062 case MVT::f32: return fastEmit_ISD_LROUND_MVT_f32_r(RetVT, Op0);
4063 case MVT::f64: return fastEmit_ISD_LROUND_MVT_f64_r(RetVT, Op0);
4064 default: return Register();
4065 }
4066}
4067
4068// FastEmit functions for ISD::SINT_TO_FP.
4069
4070Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
4071 if ((Subtarget->hasFPRCVT())) {
4072 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
4073 }
4074 if ((Subtarget->hasFullFP16())) {
4075 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
4076 }
4077 return Register();
4078}
4079
4080Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
4081 if ((Subtarget->hasFPARMv8())) {
4082 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
4083 }
4084 return Register();
4085}
4086
4087Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
4088 if ((Subtarget->hasFPRCVT())) {
4089 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
4090 }
4091 if ((Subtarget->hasFPARMv8())) {
4092 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
4093 }
4094 return Register();
4095}
4096
4097Register fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
4098switch (RetVT.SimpleTy) {
4099 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
4100 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
4101 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
4102 default: return Register();
4103}
4104}
4105
4106Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
4107 if ((Subtarget->hasFPRCVT())) {
4108 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
4109 }
4110 if ((Subtarget->hasFullFP16())) {
4111 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
4112 }
4113 return Register();
4114}
4115
4116Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
4117 if ((Subtarget->hasFPRCVT())) {
4118 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
4119 }
4120 if ((Subtarget->hasFPARMv8())) {
4121 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
4122 }
4123 return Register();
4124}
4125
4126Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
4127 if ((Subtarget->hasFPARMv8())) {
4128 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
4129 }
4130 return Register();
4131}
4132
4133Register fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
4134switch (RetVT.SimpleTy) {
4135 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
4136 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
4137 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
4138 default: return Register();
4139}
4140}
4141
4142Register fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
4143 if (RetVT.SimpleTy != MVT::v4f16)
4144 return Register();
4145 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4146 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
4147 }
4148 return Register();
4149}
4150
4151Register fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
4152 if (RetVT.SimpleTy != MVT::v8f16)
4153 return Register();
4154 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4155 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
4156 }
4157 return Register();
4158}
4159
4160Register fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
4161 if (RetVT.SimpleTy != MVT::v2f32)
4162 return Register();
4163 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4164 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
4165 }
4166 return Register();
4167}
4168
4169Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
4170 if (RetVT.SimpleTy != MVT::v4f32)
4171 return Register();
4172 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4173 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
4174 }
4175 return Register();
4176}
4177
4178Register fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
4179 if (RetVT.SimpleTy != MVT::v2f64)
4180 return Register();
4181 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4182 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
4183 }
4184 return Register();
4185}
4186
4187Register fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
4188 switch (VT.SimpleTy) {
4189 case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
4190 case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
4191 case MVT::v4i16: return fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
4192 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
4193 case MVT::v2i32: return fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
4194 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
4195 case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
4196 default: return Register();
4197 }
4198}
4199
4200// FastEmit functions for ISD::SPLAT_VECTOR.
4201
4202Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(Register Op0) {
4203 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4204 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_B, RC: &AArch64::ZPRRegClass, Op0);
4205 }
4206 return Register();
4207}
4208
4209Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(Register Op0) {
4210 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4211 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_H, RC: &AArch64::ZPRRegClass, Op0);
4212 }
4213 return Register();
4214}
4215
4216Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(Register Op0) {
4217 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4218 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_S, RC: &AArch64::ZPRRegClass, Op0);
4219 }
4220 return Register();
4221}
4222
4223Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(MVT RetVT, Register Op0) {
4224switch (RetVT.SimpleTy) {
4225 case MVT::nxv16i8: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(Op0);
4226 case MVT::nxv8i16: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(Op0);
4227 case MVT::nxv4i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(Op0);
4228 default: return Register();
4229}
4230}
4231
4232Register fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(MVT RetVT, Register Op0) {
4233 if (RetVT.SimpleTy != MVT::nxv2i64)
4234 return Register();
4235 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4236 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_D, RC: &AArch64::ZPRRegClass, Op0);
4237 }
4238 return Register();
4239}
4240
4241Register fastEmit_ISD_SPLAT_VECTOR_r(MVT VT, MVT RetVT, Register Op0) {
4242 switch (VT.SimpleTy) {
4243 case MVT::i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(RetVT, Op0);
4244 case MVT::i64: return fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(RetVT, Op0);
4245 default: return Register();
4246 }
4247}
4248
4249// FastEmit functions for ISD::STRICT_FCEIL.
4250
4251Register fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
4252 if (RetVT.SimpleTy != MVT::f16)
4253 return Register();
4254 if ((Subtarget->hasFullFP16())) {
4255 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPHr, RC: &AArch64::FPR16RegClass, Op0);
4256 }
4257 return Register();
4258}
4259
4260Register fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
4261 if (RetVT.SimpleTy != MVT::f32)
4262 return Register();
4263 if ((Subtarget->hasFPARMv8())) {
4264 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPSr, RC: &AArch64::FPR32RegClass, Op0);
4265 }
4266 return Register();
4267}
4268
4269Register fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
4270 if (RetVT.SimpleTy != MVT::f64)
4271 return Register();
4272 if ((Subtarget->hasFPARMv8())) {
4273 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPDr, RC: &AArch64::FPR64RegClass, Op0);
4274 }
4275 return Register();
4276}
4277
4278Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
4279 if (RetVT.SimpleTy != MVT::v4f16)
4280 return Register();
4281 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4282 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f16, RC: &AArch64::FPR64RegClass, Op0);
4283 }
4284 return Register();
4285}
4286
4287Register fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
4288 if (RetVT.SimpleTy != MVT::v8f16)
4289 return Register();
4290 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4291 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv8f16, RC: &AArch64::FPR128RegClass, Op0);
4292 }
4293 return Register();
4294}
4295
4296Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
4297 if (RetVT.SimpleTy != MVT::v2f32)
4298 return Register();
4299 if ((Subtarget->isNeonAvailable())) {
4300 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f32, RC: &AArch64::FPR64RegClass, Op0);
4301 }
4302 return Register();
4303}
4304
4305Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
4306 if (RetVT.SimpleTy != MVT::v4f32)
4307 return Register();
4308 if ((Subtarget->isNeonAvailable())) {
4309 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f32, RC: &AArch64::FPR128RegClass, Op0);
4310 }
4311 return Register();
4312}
4313
4314Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(MVT RetVT, Register Op0) {
4315 if (RetVT.SimpleTy != MVT::v2f64)
4316 return Register();
4317 if ((Subtarget->isNeonAvailable())) {
4318 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f64, RC: &AArch64::FPR128RegClass, Op0);
4319 }
4320 return Register();
4321}
4322
4323Register fastEmit_ISD_STRICT_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
4324 switch (VT.SimpleTy) {
4325 case MVT::f16: return fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(RetVT, Op0);
4326 case MVT::f32: return fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(RetVT, Op0);
4327 case MVT::f64: return fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(RetVT, Op0);
4328 case MVT::v4f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(RetVT, Op0);
4329 case MVT::v8f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(RetVT, Op0);
4330 case MVT::v2f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(RetVT, Op0);
4331 case MVT::v4f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(RetVT, Op0);
4332 case MVT::v2f64: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(RetVT, Op0);
4333 default: return Register();
4334 }
4335}
4336
4337// FastEmit functions for ISD::STRICT_FFLOOR.
4338
4339Register fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
4340 if (RetVT.SimpleTy != MVT::f16)
4341 return Register();
4342 if ((Subtarget->hasFullFP16())) {
4343 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMHr, RC: &AArch64::FPR16RegClass, Op0);
4344 }
4345 return Register();
4346}
4347
4348Register fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
4349 if (RetVT.SimpleTy != MVT::f32)
4350 return Register();
4351 if ((Subtarget->hasFPARMv8())) {
4352 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMSr, RC: &AArch64::FPR32RegClass, Op0);
4353 }
4354 return Register();
4355}
4356
4357Register fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
4358 if (RetVT.SimpleTy != MVT::f64)
4359 return Register();
4360 if ((Subtarget->hasFPARMv8())) {
4361 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMDr, RC: &AArch64::FPR64RegClass, Op0);
4362 }
4363 return Register();
4364}
4365
4366Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
4367 if (RetVT.SimpleTy != MVT::v4f16)
4368 return Register();
4369 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4370 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f16, RC: &AArch64::FPR64RegClass, Op0);
4371 }
4372 return Register();
4373}
4374
4375Register fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
4376 if (RetVT.SimpleTy != MVT::v8f16)
4377 return Register();
4378 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4379 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv8f16, RC: &AArch64::FPR128RegClass, Op0);
4380 }
4381 return Register();
4382}
4383
4384Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
4385 if (RetVT.SimpleTy != MVT::v2f32)
4386 return Register();
4387 if ((Subtarget->isNeonAvailable())) {
4388 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f32, RC: &AArch64::FPR64RegClass, Op0);
4389 }
4390 return Register();
4391}
4392
4393Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
4394 if (RetVT.SimpleTy != MVT::v4f32)
4395 return Register();
4396 if ((Subtarget->isNeonAvailable())) {
4397 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f32, RC: &AArch64::FPR128RegClass, Op0);
4398 }
4399 return Register();
4400}
4401
4402Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(MVT RetVT, Register Op0) {
4403 if (RetVT.SimpleTy != MVT::v2f64)
4404 return Register();
4405 if ((Subtarget->isNeonAvailable())) {
4406 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f64, RC: &AArch64::FPR128RegClass, Op0);
4407 }
4408 return Register();
4409}
4410
4411Register fastEmit_ISD_STRICT_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
4412 switch (VT.SimpleTy) {
4413 case MVT::f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(RetVT, Op0);
4414 case MVT::f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(RetVT, Op0);
4415 case MVT::f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(RetVT, Op0);
4416 case MVT::v4f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(RetVT, Op0);
4417 case MVT::v8f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(RetVT, Op0);
4418 case MVT::v2f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(RetVT, Op0);
4419 case MVT::v4f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(RetVT, Op0);
4420 case MVT::v2f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(RetVT, Op0);
4421 default: return Register();
4422 }
4423}
4424
4425// FastEmit functions for ISD::STRICT_FNEARBYINT.
4426
4427Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
4428 if (RetVT.SimpleTy != MVT::f16)
4429 return Register();
4430 if ((Subtarget->hasFullFP16())) {
4431 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIHr, RC: &AArch64::FPR16RegClass, Op0);
4432 }
4433 return Register();
4434}
4435
4436Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
4437 if (RetVT.SimpleTy != MVT::f32)
4438 return Register();
4439 if ((Subtarget->hasFPARMv8())) {
4440 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTISr, RC: &AArch64::FPR32RegClass, Op0);
4441 }
4442 return Register();
4443}
4444
4445Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
4446 if (RetVT.SimpleTy != MVT::f64)
4447 return Register();
4448 if ((Subtarget->hasFPARMv8())) {
4449 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIDr, RC: &AArch64::FPR64RegClass, Op0);
4450 }
4451 return Register();
4452}
4453
4454Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4455 if (RetVT.SimpleTy != MVT::v4f16)
4456 return Register();
4457 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4458 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f16, RC: &AArch64::FPR64RegClass, Op0);
4459 }
4460 return Register();
4461}
4462
4463Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4464 if (RetVT.SimpleTy != MVT::v8f16)
4465 return Register();
4466 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4467 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv8f16, RC: &AArch64::FPR128RegClass, Op0);
4468 }
4469 return Register();
4470}
4471
4472Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4473 if (RetVT.SimpleTy != MVT::v2f32)
4474 return Register();
4475 if ((Subtarget->isNeonAvailable())) {
4476 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f32, RC: &AArch64::FPR64RegClass, Op0);
4477 }
4478 return Register();
4479}
4480
4481Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4482 if (RetVT.SimpleTy != MVT::v4f32)
4483 return Register();
4484 if ((Subtarget->isNeonAvailable())) {
4485 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f32, RC: &AArch64::FPR128RegClass, Op0);
4486 }
4487 return Register();
4488}
4489
4490Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4491 if (RetVT.SimpleTy != MVT::v2f64)
4492 return Register();
4493 if ((Subtarget->isNeonAvailable())) {
4494 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f64, RC: &AArch64::FPR128RegClass, Op0);
4495 }
4496 return Register();
4497}
4498
4499Register fastEmit_ISD_STRICT_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
4500 switch (VT.SimpleTy) {
4501 case MVT::f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(RetVT, Op0);
4502 case MVT::f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(RetVT, Op0);
4503 case MVT::f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(RetVT, Op0);
4504 case MVT::v4f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(RetVT, Op0);
4505 case MVT::v8f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(RetVT, Op0);
4506 case MVT::v2f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(RetVT, Op0);
4507 case MVT::v4f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(RetVT, Op0);
4508 case MVT::v2f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(RetVT, Op0);
4509 default: return Register();
4510 }
4511}
4512
4513// FastEmit functions for ISD::STRICT_FP_EXTEND.
4514
4515Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(Register Op0) {
4516 if ((Subtarget->hasFPARMv8())) {
4517 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSHr, RC: &AArch64::FPR32RegClass, Op0);
4518 }
4519 return Register();
4520}
4521
4522Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(Register Op0) {
4523 if ((Subtarget->hasFPARMv8())) {
4524 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDHr, RC: &AArch64::FPR64RegClass, Op0);
4525 }
4526 return Register();
4527}
4528
4529Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(MVT RetVT, Register Op0) {
4530switch (RetVT.SimpleTy) {
4531 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(Op0);
4532 case MVT::f64: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(Op0);
4533 default: return Register();
4534}
4535}
4536
4537Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
4538 if (RetVT.SimpleTy != MVT::f64)
4539 return Register();
4540 if ((Subtarget->hasFPARMv8())) {
4541 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDSr, RC: &AArch64::FPR64RegClass, Op0);
4542 }
4543 return Register();
4544}
4545
4546Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
4547 if (RetVT.SimpleTy != MVT::v4f32)
4548 return Register();
4549 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv4i16, RC: &AArch64::FPR128RegClass, Op0);
4550}
4551
4552Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4bf16_r(MVT RetVT, Register Op0) {
4553 if (RetVT.SimpleTy != MVT::v4f32)
4554 return Register();
4555 if ((Subtarget->isNeonAvailable())) {
4556 return fastEmitInst_r(MachineInstOpcode: AArch64::SHLLv4i16, RC: &AArch64::FPR128RegClass, Op0);
4557 }
4558 return Register();
4559}
4560
4561Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(MVT RetVT, Register Op0) {
4562 if (RetVT.SimpleTy != MVT::v2f64)
4563 return Register();
4564 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv2i32, RC: &AArch64::FPR128RegClass, Op0);
4565}
4566
4567Register fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
4568 switch (VT.SimpleTy) {
4569 case MVT::f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(RetVT, Op0);
4570 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0);
4571 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
4572 case MVT::v4bf16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4bf16_r(RetVT, Op0);
4573 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(RetVT, Op0);
4574 default: return Register();
4575 }
4576}
4577
4578// FastEmit functions for ISD::STRICT_FP_ROUND.
4579
4580Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_bf16_r(Register Op0) {
4581 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
4582 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVT, RC: &AArch64::FPR16RegClass, Op0);
4583 }
4584 return Register();
4585}
4586
4587Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_f16_r(Register Op0) {
4588 if ((Subtarget->hasFPARMv8())) {
4589 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHSr, RC: &AArch64::FPR16RegClass, Op0);
4590 }
4591 return Register();
4592}
4593
4594Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4595switch (RetVT.SimpleTy) {
4596 case MVT::bf16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_bf16_r(Op0);
4597 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_f16_r(Op0);
4598 default: return Register();
4599}
4600}
4601
4602Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(Register Op0) {
4603 if ((Subtarget->hasFPARMv8())) {
4604 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHDr, RC: &AArch64::FPR16RegClass, Op0);
4605 }
4606 return Register();
4607}
4608
4609Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(Register Op0) {
4610 if ((Subtarget->hasFPARMv8())) {
4611 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSDr, RC: &AArch64::FPR32RegClass, Op0);
4612 }
4613 return Register();
4614}
4615
4616Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4617switch (RetVT.SimpleTy) {
4618 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(Op0);
4619 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(Op0);
4620 default: return Register();
4621}
4622}
4623
4624Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Register Op0) {
4625 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
4626}
4627
4628Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Register Op0) {
4629 if ((Subtarget->hasBF16()) && (Subtarget->isNeonAvailable())) {
4630 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVTN, RC: &AArch64::FPR64RegClass, Op0);
4631 }
4632 return Register();
4633}
4634
4635Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
4636switch (RetVT.SimpleTy) {
4637 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Op0);
4638 case MVT::v4bf16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Op0);
4639 default: return Register();
4640}
4641}
4642
4643Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
4644 if (RetVT.SimpleTy != MVT::v2f32)
4645 return Register();
4646 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
4647}
4648
4649Register fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
4650 switch (VT.SimpleTy) {
4651 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(RetVT, Op0);
4652 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0);
4653 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
4654 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(RetVT, Op0);
4655 default: return Register();
4656 }
4657}
4658
4659// FastEmit functions for ISD::STRICT_FP_TO_SINT.
4660
4661Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
4662 if ((Subtarget->hasFPRCVT())) {
4663 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSHr, RC: &AArch64::FPR32RegClass, Op0);
4664 }
4665 if ((Subtarget->hasFullFP16())) {
4666 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
4667 }
4668 return Register();
4669}
4670
4671Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
4672 if ((Subtarget->hasFPRCVT())) {
4673 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDHr, RC: &AArch64::FPR64RegClass, Op0);
4674 }
4675 if ((Subtarget->hasFullFP16())) {
4676 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
4677 }
4678 return Register();
4679}
4680
4681Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
4682switch (RetVT.SimpleTy) {
4683 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
4684 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
4685 default: return Register();
4686}
4687}
4688
4689Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
4690 if ((Subtarget->hasFPARMv8())) {
4691 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
4692 }
4693 return Register();
4694}
4695
4696Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
4697 if ((Subtarget->hasFPRCVT())) {
4698 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDSr, RC: &AArch64::FPR64RegClass, Op0);
4699 }
4700 if ((Subtarget->hasFPARMv8())) {
4701 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
4702 }
4703 return Register();
4704}
4705
4706Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
4707switch (RetVT.SimpleTy) {
4708 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
4709 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
4710 default: return Register();
4711}
4712}
4713
4714Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
4715 if ((Subtarget->hasFPRCVT())) {
4716 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSDr, RC: &AArch64::FPR32RegClass, Op0);
4717 }
4718 if ((Subtarget->hasFPARMv8())) {
4719 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
4720 }
4721 return Register();
4722}
4723
4724Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
4725 if ((Subtarget->hasFPARMv8())) {
4726 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
4727 }
4728 return Register();
4729}
4730
4731Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
4732switch (RetVT.SimpleTy) {
4733 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
4734 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
4735 default: return Register();
4736}
4737}
4738
4739Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4740 if (RetVT.SimpleTy != MVT::v4i16)
4741 return Register();
4742 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4743 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
4744 }
4745 return Register();
4746}
4747
4748Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4749 if (RetVT.SimpleTy != MVT::v8i16)
4750 return Register();
4751 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4752 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
4753 }
4754 return Register();
4755}
4756
4757Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4758 if (RetVT.SimpleTy != MVT::v2i32)
4759 return Register();
4760 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4761 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
4762 }
4763 return Register();
4764}
4765
4766Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4767 if (RetVT.SimpleTy != MVT::v4i32)
4768 return Register();
4769 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4770 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
4771 }
4772 return Register();
4773}
4774
4775Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4776 if (RetVT.SimpleTy != MVT::v2i64)
4777 return Register();
4778 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4779 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
4780 }
4781 return Register();
4782}
4783
4784Register fastEmit_ISD_STRICT_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
4785 switch (VT.SimpleTy) {
4786 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
4787 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
4788 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
4789 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
4790 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
4791 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
4792 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
4793 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0);
4794 default: return Register();
4795 }
4796}
4797
4798// FastEmit functions for ISD::STRICT_FP_TO_UINT.
4799
4800Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
4801 if ((Subtarget->hasFPRCVT())) {
4802 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSHr, RC: &AArch64::FPR32RegClass, Op0);
4803 }
4804 if ((Subtarget->hasFullFP16())) {
4805 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
4806 }
4807 return Register();
4808}
4809
4810Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
4811 if ((Subtarget->hasFPRCVT())) {
4812 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDHr, RC: &AArch64::FPR64RegClass, Op0);
4813 }
4814 if ((Subtarget->hasFullFP16())) {
4815 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
4816 }
4817 return Register();
4818}
4819
4820Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
4821switch (RetVT.SimpleTy) {
4822 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
4823 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
4824 default: return Register();
4825}
4826}
4827
4828Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
4829 if ((Subtarget->hasFPARMv8())) {
4830 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
4831 }
4832 return Register();
4833}
4834
4835Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
4836 if ((Subtarget->hasFPRCVT())) {
4837 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDSr, RC: &AArch64::FPR64RegClass, Op0);
4838 }
4839 if ((Subtarget->hasFPARMv8())) {
4840 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
4841 }
4842 return Register();
4843}
4844
4845Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
4846switch (RetVT.SimpleTy) {
4847 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
4848 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
4849 default: return Register();
4850}
4851}
4852
4853Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
4854 if ((Subtarget->hasFPRCVT())) {
4855 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSDr, RC: &AArch64::FPR32RegClass, Op0);
4856 }
4857 if ((Subtarget->hasFPARMv8())) {
4858 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
4859 }
4860 return Register();
4861}
4862
4863Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
4864 if ((Subtarget->hasFPARMv8())) {
4865 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
4866 }
4867 return Register();
4868}
4869
4870Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
4871switch (RetVT.SimpleTy) {
4872 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
4873 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
4874 default: return Register();
4875}
4876}
4877
4878Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4879 if (RetVT.SimpleTy != MVT::v4i16)
4880 return Register();
4881 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4882 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
4883 }
4884 return Register();
4885}
4886
4887Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4888 if (RetVT.SimpleTy != MVT::v8i16)
4889 return Register();
4890 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4891 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
4892 }
4893 return Register();
4894}
4895
4896Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4897 if (RetVT.SimpleTy != MVT::v2i32)
4898 return Register();
4899 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4900 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
4901 }
4902 return Register();
4903}
4904
4905Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4906 if (RetVT.SimpleTy != MVT::v4i32)
4907 return Register();
4908 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4909 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
4910 }
4911 return Register();
4912}
4913
4914Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4915 if (RetVT.SimpleTy != MVT::v2i64)
4916 return Register();
4917 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4918 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
4919 }
4920 return Register();
4921}
4922
4923Register fastEmit_ISD_STRICT_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
4924 switch (VT.SimpleTy) {
4925 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
4926 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
4927 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
4928 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
4929 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
4930 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
4931 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
4932 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0);
4933 default: return Register();
4934 }
4935}
4936
4937// FastEmit functions for ISD::STRICT_FRINT.
4938
4939Register fastEmit_ISD_STRICT_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
4940 if (RetVT.SimpleTy != MVT::f16)
4941 return Register();
4942 if ((Subtarget->hasFullFP16())) {
4943 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXHr, RC: &AArch64::FPR16RegClass, Op0);
4944 }
4945 return Register();
4946}
4947
4948Register fastEmit_ISD_STRICT_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
4949 if (RetVT.SimpleTy != MVT::f32)
4950 return Register();
4951 if ((Subtarget->hasFPARMv8())) {
4952 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXSr, RC: &AArch64::FPR32RegClass, Op0);
4953 }
4954 return Register();
4955}
4956
4957Register fastEmit_ISD_STRICT_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
4958 if (RetVT.SimpleTy != MVT::f64)
4959 return Register();
4960 if ((Subtarget->hasFPARMv8())) {
4961 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXDr, RC: &AArch64::FPR64RegClass, Op0);
4962 }
4963 return Register();
4964}
4965
4966Register fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4967 if (RetVT.SimpleTy != MVT::v4f16)
4968 return Register();
4969 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4970 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f16, RC: &AArch64::FPR64RegClass, Op0);
4971 }
4972 return Register();
4973}
4974
4975Register fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4976 if (RetVT.SimpleTy != MVT::v8f16)
4977 return Register();
4978 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4979 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv8f16, RC: &AArch64::FPR128RegClass, Op0);
4980 }
4981 return Register();
4982}
4983
4984Register fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4985 if (RetVT.SimpleTy != MVT::v2f32)
4986 return Register();
4987 if ((Subtarget->isNeonAvailable())) {
4988 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f32, RC: &AArch64::FPR64RegClass, Op0);
4989 }
4990 return Register();
4991}
4992
4993Register fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4994 if (RetVT.SimpleTy != MVT::v4f32)
4995 return Register();
4996 if ((Subtarget->isNeonAvailable())) {
4997 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f32, RC: &AArch64::FPR128RegClass, Op0);
4998 }
4999 return Register();
5000}
5001
5002Register fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
5003 if (RetVT.SimpleTy != MVT::v2f64)
5004 return Register();
5005 if ((Subtarget->isNeonAvailable())) {
5006 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f64, RC: &AArch64::FPR128RegClass, Op0);
5007 }
5008 return Register();
5009}
5010
5011Register fastEmit_ISD_STRICT_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
5012 switch (VT.SimpleTy) {
5013 case MVT::f16: return fastEmit_ISD_STRICT_FRINT_MVT_f16_r(RetVT, Op0);
5014 case MVT::f32: return fastEmit_ISD_STRICT_FRINT_MVT_f32_r(RetVT, Op0);
5015 case MVT::f64: return fastEmit_ISD_STRICT_FRINT_MVT_f64_r(RetVT, Op0);
5016 case MVT::v4f16: return fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(RetVT, Op0);
5017 case MVT::v8f16: return fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(RetVT, Op0);
5018 case MVT::v2f32: return fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(RetVT, Op0);
5019 case MVT::v4f32: return fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(RetVT, Op0);
5020 case MVT::v2f64: return fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(RetVT, Op0);
5021 default: return Register();
5022 }
5023}
5024
5025// FastEmit functions for ISD::STRICT_FROUND.
5026
5027Register fastEmit_ISD_STRICT_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5028 if (RetVT.SimpleTy != MVT::f16)
5029 return Register();
5030 if ((Subtarget->hasFullFP16())) {
5031 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAHr, RC: &AArch64::FPR16RegClass, Op0);
5032 }
5033 return Register();
5034}
5035
5036Register fastEmit_ISD_STRICT_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5037 if (RetVT.SimpleTy != MVT::f32)
5038 return Register();
5039 if ((Subtarget->hasFPARMv8())) {
5040 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTASr, RC: &AArch64::FPR32RegClass, Op0);
5041 }
5042 return Register();
5043}
5044
5045Register fastEmit_ISD_STRICT_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5046 if (RetVT.SimpleTy != MVT::f64)
5047 return Register();
5048 if ((Subtarget->hasFPARMv8())) {
5049 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTADr, RC: &AArch64::FPR64RegClass, Op0);
5050 }
5051 return Register();
5052}
5053
5054Register fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
5055 if (RetVT.SimpleTy != MVT::v4f16)
5056 return Register();
5057 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5058 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f16, RC: &AArch64::FPR64RegClass, Op0);
5059 }
5060 return Register();
5061}
5062
5063Register fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
5064 if (RetVT.SimpleTy != MVT::v8f16)
5065 return Register();
5066 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5067 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv8f16, RC: &AArch64::FPR128RegClass, Op0);
5068 }
5069 return Register();
5070}
5071
5072Register fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
5073 if (RetVT.SimpleTy != MVT::v2f32)
5074 return Register();
5075 if ((Subtarget->isNeonAvailable())) {
5076 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f32, RC: &AArch64::FPR64RegClass, Op0);
5077 }
5078 return Register();
5079}
5080
5081Register fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
5082 if (RetVT.SimpleTy != MVT::v4f32)
5083 return Register();
5084 if ((Subtarget->isNeonAvailable())) {
5085 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f32, RC: &AArch64::FPR128RegClass, Op0);
5086 }
5087 return Register();
5088}
5089
5090Register fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
5091 if (RetVT.SimpleTy != MVT::v2f64)
5092 return Register();
5093 if ((Subtarget->isNeonAvailable())) {
5094 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f64, RC: &AArch64::FPR128RegClass, Op0);
5095 }
5096 return Register();
5097}
5098
5099Register fastEmit_ISD_STRICT_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
5100 switch (VT.SimpleTy) {
5101 case MVT::f16: return fastEmit_ISD_STRICT_FROUND_MVT_f16_r(RetVT, Op0);
5102 case MVT::f32: return fastEmit_ISD_STRICT_FROUND_MVT_f32_r(RetVT, Op0);
5103 case MVT::f64: return fastEmit_ISD_STRICT_FROUND_MVT_f64_r(RetVT, Op0);
5104 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(RetVT, Op0);
5105 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(RetVT, Op0);
5106 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(RetVT, Op0);
5107 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(RetVT, Op0);
5108 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(RetVT, Op0);
5109 default: return Register();
5110 }
5111}
5112
5113// FastEmit functions for ISD::STRICT_FROUNDEVEN.
5114
5115Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
5116 if (RetVT.SimpleTy != MVT::f16)
5117 return Register();
5118 if ((Subtarget->hasFullFP16())) {
5119 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNHr, RC: &AArch64::FPR16RegClass, Op0);
5120 }
5121 return Register();
5122}
5123
5124Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
5125 if (RetVT.SimpleTy != MVT::f32)
5126 return Register();
5127 if ((Subtarget->hasFPARMv8())) {
5128 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNSr, RC: &AArch64::FPR32RegClass, Op0);
5129 }
5130 return Register();
5131}
5132
5133Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
5134 if (RetVT.SimpleTy != MVT::f64)
5135 return Register();
5136 if ((Subtarget->hasFPARMv8())) {
5137 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNDr, RC: &AArch64::FPR64RegClass, Op0);
5138 }
5139 return Register();
5140}
5141
5142Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
5143 if (RetVT.SimpleTy != MVT::v4f16)
5144 return Register();
5145 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5146 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f16, RC: &AArch64::FPR64RegClass, Op0);
5147 }
5148 return Register();
5149}
5150
5151Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
5152 if (RetVT.SimpleTy != MVT::v8f16)
5153 return Register();
5154 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5155 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv8f16, RC: &AArch64::FPR128RegClass, Op0);
5156 }
5157 return Register();
5158}
5159
5160Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
5161 if (RetVT.SimpleTy != MVT::v2f32)
5162 return Register();
5163 if ((Subtarget->isNeonAvailable())) {
5164 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f32, RC: &AArch64::FPR64RegClass, Op0);
5165 }
5166 return Register();
5167}
5168
5169Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
5170 if (RetVT.SimpleTy != MVT::v4f32)
5171 return Register();
5172 if ((Subtarget->isNeonAvailable())) {
5173 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f32, RC: &AArch64::FPR128RegClass, Op0);
5174 }
5175 return Register();
5176}
5177
5178Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, Register Op0) {
5179 if (RetVT.SimpleTy != MVT::v2f64)
5180 return Register();
5181 if ((Subtarget->isNeonAvailable())) {
5182 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f64, RC: &AArch64::FPR128RegClass, Op0);
5183 }
5184 return Register();
5185}
5186
5187Register fastEmit_ISD_STRICT_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
5188 switch (VT.SimpleTy) {
5189 case MVT::f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
5190 case MVT::f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
5191 case MVT::f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
5192 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
5193 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
5194 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
5195 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
5196 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0);
5197 default: return Register();
5198 }
5199}
5200
5201// FastEmit functions for ISD::STRICT_FSQRT.
5202
5203Register fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
5204 if (RetVT.SimpleTy != MVT::f16)
5205 return Register();
5206 if ((Subtarget->hasFullFP16())) {
5207 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTHr, RC: &AArch64::FPR16RegClass, Op0);
5208 }
5209 return Register();
5210}
5211
5212Register fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
5213 if (RetVT.SimpleTy != MVT::f32)
5214 return Register();
5215 if ((Subtarget->hasFPARMv8())) {
5216 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTSr, RC: &AArch64::FPR32RegClass, Op0);
5217 }
5218 return Register();
5219}
5220
5221Register fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
5222 if (RetVT.SimpleTy != MVT::f64)
5223 return Register();
5224 if ((Subtarget->hasFPARMv8())) {
5225 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTDr, RC: &AArch64::FPR64RegClass, Op0);
5226 }
5227 return Register();
5228}
5229
5230Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(MVT RetVT, Register Op0) {
5231 if (RetVT.SimpleTy != MVT::v4f16)
5232 return Register();
5233 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5234 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f16, RC: &AArch64::FPR64RegClass, Op0);
5235 }
5236 return Register();
5237}
5238
5239Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
5240 if (RetVT.SimpleTy != MVT::v8f16)
5241 return Register();
5242 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5243 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv8f16, RC: &AArch64::FPR128RegClass, Op0);
5244 }
5245 return Register();
5246}
5247
5248Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(MVT RetVT, Register Op0) {
5249 if (RetVT.SimpleTy != MVT::v2f32)
5250 return Register();
5251 if ((Subtarget->isNeonAvailable())) {
5252 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f32, RC: &AArch64::FPR64RegClass, Op0);
5253 }
5254 return Register();
5255}
5256
5257Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
5258 if (RetVT.SimpleTy != MVT::v4f32)
5259 return Register();
5260 if ((Subtarget->isNeonAvailable())) {
5261 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f32, RC: &AArch64::FPR128RegClass, Op0);
5262 }
5263 return Register();
5264}
5265
5266Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
5267 if (RetVT.SimpleTy != MVT::v2f64)
5268 return Register();
5269 if ((Subtarget->isNeonAvailable())) {
5270 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f64, RC: &AArch64::FPR128RegClass, Op0);
5271 }
5272 return Register();
5273}
5274
5275Register fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
5276 switch (VT.SimpleTy) {
5277 case MVT::f16: return fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(RetVT, Op0);
5278 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0);
5279 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0);
5280 case MVT::v4f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(RetVT, Op0);
5281 case MVT::v8f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(RetVT, Op0);
5282 case MVT::v2f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(RetVT, Op0);
5283 case MVT::v4f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(RetVT, Op0);
5284 case MVT::v2f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(RetVT, Op0);
5285 default: return Register();
5286 }
5287}
5288
5289// FastEmit functions for ISD::STRICT_FTRUNC.
5290
5291Register fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
5292 if (RetVT.SimpleTy != MVT::f16)
5293 return Register();
5294 if ((Subtarget->hasFullFP16())) {
5295 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZHr, RC: &AArch64::FPR16RegClass, Op0);
5296 }
5297 return Register();
5298}
5299
5300Register fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
5301 if (RetVT.SimpleTy != MVT::f32)
5302 return Register();
5303 if ((Subtarget->hasFPARMv8())) {
5304 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZSr, RC: &AArch64::FPR32RegClass, Op0);
5305 }
5306 return Register();
5307}
5308
5309Register fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
5310 if (RetVT.SimpleTy != MVT::f64)
5311 return Register();
5312 if ((Subtarget->hasFPARMv8())) {
5313 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZDr, RC: &AArch64::FPR64RegClass, Op0);
5314 }
5315 return Register();
5316}
5317
5318Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
5319 if (RetVT.SimpleTy != MVT::v4f16)
5320 return Register();
5321 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5322 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f16, RC: &AArch64::FPR64RegClass, Op0);
5323 }
5324 return Register();
5325}
5326
5327Register fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
5328 if (RetVT.SimpleTy != MVT::v8f16)
5329 return Register();
5330 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5331 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv8f16, RC: &AArch64::FPR128RegClass, Op0);
5332 }
5333 return Register();
5334}
5335
5336Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
5337 if (RetVT.SimpleTy != MVT::v2f32)
5338 return Register();
5339 if ((Subtarget->isNeonAvailable())) {
5340 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f32, RC: &AArch64::FPR64RegClass, Op0);
5341 }
5342 return Register();
5343}
5344
5345Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
5346 if (RetVT.SimpleTy != MVT::v4f32)
5347 return Register();
5348 if ((Subtarget->isNeonAvailable())) {
5349 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f32, RC: &AArch64::FPR128RegClass, Op0);
5350 }
5351 return Register();
5352}
5353
5354Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(MVT RetVT, Register Op0) {
5355 if (RetVT.SimpleTy != MVT::v2f64)
5356 return Register();
5357 if ((Subtarget->isNeonAvailable())) {
5358 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f64, RC: &AArch64::FPR128RegClass, Op0);
5359 }
5360 return Register();
5361}
5362
5363Register fastEmit_ISD_STRICT_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
5364 switch (VT.SimpleTy) {
5365 case MVT::f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(RetVT, Op0);
5366 case MVT::f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(RetVT, Op0);
5367 case MVT::f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(RetVT, Op0);
5368 case MVT::v4f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(RetVT, Op0);
5369 case MVT::v8f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(RetVT, Op0);
5370 case MVT::v2f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(RetVT, Op0);
5371 case MVT::v4f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(RetVT, Op0);
5372 case MVT::v2f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(RetVT, Op0);
5373 default: return Register();
5374 }
5375}
5376
5377// FastEmit functions for ISD::STRICT_LLROUND.
5378
5379Register fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5380 if (RetVT.SimpleTy != MVT::i64)
5381 return Register();
5382 if ((Subtarget->hasFullFP16())) {
5383 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
5384 }
5385 return Register();
5386}
5387
5388Register fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5389 if (RetVT.SimpleTy != MVT::i64)
5390 return Register();
5391 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
5392}
5393
5394Register fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5395 if (RetVT.SimpleTy != MVT::i64)
5396 return Register();
5397 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
5398}
5399
5400Register fastEmit_ISD_STRICT_LLROUND_r(MVT VT, MVT RetVT, Register Op0) {
5401 switch (VT.SimpleTy) {
5402 case MVT::f16: return fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(RetVT, Op0);
5403 case MVT::f32: return fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(RetVT, Op0);
5404 case MVT::f64: return fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(RetVT, Op0);
5405 default: return Register();
5406 }
5407}
5408
5409// FastEmit functions for ISD::STRICT_LROUND.
5410
5411Register fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(Register Op0) {
5412 if ((Subtarget->hasFullFP16())) {
5413 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWHr, RC: &AArch64::GPR32RegClass, Op0);
5414 }
5415 return Register();
5416}
5417
5418Register fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(Register Op0) {
5419 if ((Subtarget->hasFullFP16())) {
5420 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
5421 }
5422 return Register();
5423}
5424
5425Register fastEmit_ISD_STRICT_LROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5426switch (RetVT.SimpleTy) {
5427 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(Op0);
5428 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(Op0);
5429 default: return Register();
5430}
5431}
5432
5433Register fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(Register Op0) {
5434 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWSr, RC: &AArch64::GPR32RegClass, Op0);
5435}
5436
5437Register fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(Register Op0) {
5438 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
5439}
5440
5441Register fastEmit_ISD_STRICT_LROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5442switch (RetVT.SimpleTy) {
5443 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(Op0);
5444 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(Op0);
5445 default: return Register();
5446}
5447}
5448
5449Register fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(Register Op0) {
5450 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWDr, RC: &AArch64::GPR32RegClass, Op0);
5451}
5452
5453Register fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(Register Op0) {
5454 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
5455}
5456
5457Register fastEmit_ISD_STRICT_LROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5458switch (RetVT.SimpleTy) {
5459 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(Op0);
5460 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(Op0);
5461 default: return Register();
5462}
5463}
5464
5465Register fastEmit_ISD_STRICT_LROUND_r(MVT VT, MVT RetVT, Register Op0) {
5466 switch (VT.SimpleTy) {
5467 case MVT::f16: return fastEmit_ISD_STRICT_LROUND_MVT_f16_r(RetVT, Op0);
5468 case MVT::f32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_r(RetVT, Op0);
5469 case MVT::f64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_r(RetVT, Op0);
5470 default: return Register();
5471 }
5472}
5473
5474// FastEmit functions for ISD::STRICT_SINT_TO_FP.
5475
5476Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5477 if ((Subtarget->hasFPRCVT())) {
5478 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5479 }
5480 if ((Subtarget->hasFullFP16())) {
5481 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5482 }
5483 return Register();
5484}
5485
5486Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5487 if ((Subtarget->hasFPARMv8())) {
5488 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5489 }
5490 return Register();
5491}
5492
5493Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5494 if ((Subtarget->hasFPRCVT())) {
5495 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5496 }
5497 if ((Subtarget->hasFPARMv8())) {
5498 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5499 }
5500 return Register();
5501}
5502
5503Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5504switch (RetVT.SimpleTy) {
5505 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5506 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5507 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5508 default: return Register();
5509}
5510}
5511
5512Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5513 if ((Subtarget->hasFPRCVT())) {
5514 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5515 }
5516 if ((Subtarget->hasFullFP16())) {
5517 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5518 }
5519 return Register();
5520}
5521
5522Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5523 if ((Subtarget->hasFPRCVT())) {
5524 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5525 }
5526 if ((Subtarget->hasFPARMv8())) {
5527 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5528 }
5529 return Register();
5530}
5531
5532Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5533 if ((Subtarget->hasFPARMv8())) {
5534 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5535 }
5536 return Register();
5537}
5538
5539Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5540switch (RetVT.SimpleTy) {
5541 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5542 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5543 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5544 default: return Register();
5545}
5546}
5547
5548Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5549 if (RetVT.SimpleTy != MVT::v4f16)
5550 return Register();
5551 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5552 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5553 }
5554 return Register();
5555}
5556
5557Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5558 if (RetVT.SimpleTy != MVT::v8f16)
5559 return Register();
5560 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5561 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5562 }
5563 return Register();
5564}
5565
5566Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5567 if (RetVT.SimpleTy != MVT::v2f32)
5568 return Register();
5569 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5570 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5571 }
5572 return Register();
5573}
5574
5575Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5576 if (RetVT.SimpleTy != MVT::v4f32)
5577 return Register();
5578 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5579 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5580 }
5581 return Register();
5582}
5583
5584Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5585 if (RetVT.SimpleTy != MVT::v2f64)
5586 return Register();
5587 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5588 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5589 }
5590 return Register();
5591}
5592
5593Register fastEmit_ISD_STRICT_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
5594 switch (VT.SimpleTy) {
5595 case MVT::i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
5596 case MVT::i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
5597 case MVT::v4i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
5598 case MVT::v8i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
5599 case MVT::v2i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
5600 case MVT::v4i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
5601 case MVT::v2i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
5602 default: return Register();
5603 }
5604}
5605
5606// FastEmit functions for ISD::STRICT_UINT_TO_FP.
5607
5608Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5609 if ((Subtarget->hasFPRCVT())) {
5610 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5611 }
5612 if ((Subtarget->hasFullFP16())) {
5613 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5614 }
5615 return Register();
5616}
5617
5618Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5619 if ((Subtarget->hasFPARMv8())) {
5620 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5621 }
5622 return Register();
5623}
5624
5625Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5626 if ((Subtarget->hasFPRCVT())) {
5627 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5628 }
5629 if ((Subtarget->hasFPARMv8())) {
5630 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5631 }
5632 return Register();
5633}
5634
5635Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5636switch (RetVT.SimpleTy) {
5637 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5638 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5639 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5640 default: return Register();
5641}
5642}
5643
5644Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5645 if ((Subtarget->hasFPRCVT())) {
5646 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5647 }
5648 if ((Subtarget->hasFullFP16())) {
5649 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5650 }
5651 return Register();
5652}
5653
5654Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5655 if ((Subtarget->hasFPRCVT())) {
5656 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5657 }
5658 if ((Subtarget->hasFPARMv8())) {
5659 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5660 }
5661 return Register();
5662}
5663
5664Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5665 if ((Subtarget->hasFPARMv8())) {
5666 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5667 }
5668 return Register();
5669}
5670
5671Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5672switch (RetVT.SimpleTy) {
5673 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5674 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5675 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5676 default: return Register();
5677}
5678}
5679
5680Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5681 if (RetVT.SimpleTy != MVT::v4f16)
5682 return Register();
5683 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5684 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5685 }
5686 return Register();
5687}
5688
5689Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5690 if (RetVT.SimpleTy != MVT::v8f16)
5691 return Register();
5692 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5693 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5694 }
5695 return Register();
5696}
5697
5698Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5699 if (RetVT.SimpleTy != MVT::v2f32)
5700 return Register();
5701 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5702 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5703 }
5704 return Register();
5705}
5706
5707Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5708 if (RetVT.SimpleTy != MVT::v4f32)
5709 return Register();
5710 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5711 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5712 }
5713 return Register();
5714}
5715
5716Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5717 if (RetVT.SimpleTy != MVT::v2f64)
5718 return Register();
5719 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5720 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5721 }
5722 return Register();
5723}
5724
5725Register fastEmit_ISD_STRICT_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
5726 switch (VT.SimpleTy) {
5727 case MVT::i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(RetVT, Op0);
5728 case MVT::i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(RetVT, Op0);
5729 case MVT::v4i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
5730 case MVT::v8i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
5731 case MVT::v2i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
5732 case MVT::v4i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
5733 case MVT::v2i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
5734 default: return Register();
5735 }
5736}
5737
5738// FastEmit functions for ISD::TRUNCATE.
5739
5740Register fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, Register Op0) {
5741 if (RetVT.SimpleTy != MVT::i32)
5742 return Register();
5743 return fastEmitInst_extractsubreg(RetVT, Op0, Idx: AArch64::sub_32);
5744}
5745
5746Register fastEmit_ISD_TRUNCATE_MVT_v8i16_r(MVT RetVT, Register Op0) {
5747 if (RetVT.SimpleTy != MVT::v8i8)
5748 return Register();
5749 if ((Subtarget->isNeonAvailable())) {
5750 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5751 }
5752 return Register();
5753}
5754
5755Register fastEmit_ISD_TRUNCATE_MVT_v4i32_r(MVT RetVT, Register Op0) {
5756 if (RetVT.SimpleTy != MVT::v4i16)
5757 return Register();
5758 if ((Subtarget->isNeonAvailable())) {
5759 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5760 }
5761 return Register();
5762}
5763
5764Register fastEmit_ISD_TRUNCATE_MVT_v2i64_r(MVT RetVT, Register Op0) {
5765 if (RetVT.SimpleTy != MVT::v2i32)
5766 return Register();
5767 if ((Subtarget->isNeonAvailable())) {
5768 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5769 }
5770 return Register();
5771}
5772
5773Register fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, Register Op0) {
5774 switch (VT.SimpleTy) {
5775 case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0);
5776 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i16_r(RetVT, Op0);
5777 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_MVT_v4i32_r(RetVT, Op0);
5778 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_MVT_v2i64_r(RetVT, Op0);
5779 default: return Register();
5780 }
5781}
5782
5783// FastEmit functions for ISD::TRUNCATE_SSAT_S.
5784
5785Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v8i16_r(MVT RetVT, Register Op0) {
5786 if (RetVT.SimpleTy != MVT::v8i8)
5787 return Register();
5788 if ((Subtarget->isNeonAvailable())) {
5789 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5790 }
5791 return Register();
5792}
5793
5794Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v4i32_r(MVT RetVT, Register Op0) {
5795 if (RetVT.SimpleTy != MVT::v4i16)
5796 return Register();
5797 if ((Subtarget->isNeonAvailable())) {
5798 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5799 }
5800 return Register();
5801}
5802
5803Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v2i64_r(MVT RetVT, Register Op0) {
5804 if (RetVT.SimpleTy != MVT::v2i32)
5805 return Register();
5806 if ((Subtarget->isNeonAvailable())) {
5807 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5808 }
5809 return Register();
5810}
5811
5812Register fastEmit_ISD_TRUNCATE_SSAT_S_r(MVT VT, MVT RetVT, Register Op0) {
5813 switch (VT.SimpleTy) {
5814 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v8i16_r(RetVT, Op0);
5815 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v4i32_r(RetVT, Op0);
5816 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v2i64_r(RetVT, Op0);
5817 default: return Register();
5818 }
5819}
5820
5821// FastEmit functions for ISD::TRUNCATE_SSAT_U.
5822
5823Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v8i16_r(MVT RetVT, Register Op0) {
5824 if (RetVT.SimpleTy != MVT::v8i8)
5825 return Register();
5826 if ((Subtarget->isNeonAvailable())) {
5827 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5828 }
5829 return Register();
5830}
5831
5832Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v4i32_r(MVT RetVT, Register Op0) {
5833 if (RetVT.SimpleTy != MVT::v4i16)
5834 return Register();
5835 if ((Subtarget->isNeonAvailable())) {
5836 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5837 }
5838 return Register();
5839}
5840
5841Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v2i64_r(MVT RetVT, Register Op0) {
5842 if (RetVT.SimpleTy != MVT::v2i32)
5843 return Register();
5844 if ((Subtarget->isNeonAvailable())) {
5845 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5846 }
5847 return Register();
5848}
5849
5850Register fastEmit_ISD_TRUNCATE_SSAT_U_r(MVT VT, MVT RetVT, Register Op0) {
5851 switch (VT.SimpleTy) {
5852 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v8i16_r(RetVT, Op0);
5853 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v4i32_r(RetVT, Op0);
5854 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v2i64_r(RetVT, Op0);
5855 default: return Register();
5856 }
5857}
5858
5859// FastEmit functions for ISD::TRUNCATE_USAT_U.
5860
5861Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v8i16_r(MVT RetVT, Register Op0) {
5862 if (RetVT.SimpleTy != MVT::v8i8)
5863 return Register();
5864 if ((Subtarget->isNeonAvailable())) {
5865 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5866 }
5867 return Register();
5868}
5869
5870Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v4i32_r(MVT RetVT, Register Op0) {
5871 if (RetVT.SimpleTy != MVT::v4i16)
5872 return Register();
5873 if ((Subtarget->isNeonAvailable())) {
5874 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5875 }
5876 return Register();
5877}
5878
5879Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v2i64_r(MVT RetVT, Register Op0) {
5880 if (RetVT.SimpleTy != MVT::v2i32)
5881 return Register();
5882 if ((Subtarget->isNeonAvailable())) {
5883 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5884 }
5885 return Register();
5886}
5887
5888Register fastEmit_ISD_TRUNCATE_USAT_U_r(MVT VT, MVT RetVT, Register Op0) {
5889 switch (VT.SimpleTy) {
5890 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v8i16_r(RetVT, Op0);
5891 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v4i32_r(RetVT, Op0);
5892 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v2i64_r(RetVT, Op0);
5893 default: return Register();
5894 }
5895}
5896
5897// FastEmit functions for ISD::UINT_TO_FP.
5898
5899Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5900 if ((Subtarget->hasFPRCVT())) {
5901 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5902 }
5903 if ((Subtarget->hasFullFP16())) {
5904 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5905 }
5906 return Register();
5907}
5908
5909Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5910 if ((Subtarget->hasFPARMv8())) {
5911 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5912 }
5913 return Register();
5914}
5915
5916Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5917 if ((Subtarget->hasFPRCVT())) {
5918 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5919 }
5920 if ((Subtarget->hasFPARMv8())) {
5921 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5922 }
5923 return Register();
5924}
5925
5926Register fastEmit_ISD_UINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5927switch (RetVT.SimpleTy) {
5928 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5929 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5930 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5931 default: return Register();
5932}
5933}
5934
5935Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5936 if ((Subtarget->hasFPRCVT())) {
5937 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5938 }
5939 if ((Subtarget->hasFullFP16())) {
5940 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5941 }
5942 return Register();
5943}
5944
5945Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5946 if ((Subtarget->hasFPRCVT())) {
5947 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5948 }
5949 if ((Subtarget->hasFPARMv8())) {
5950 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5951 }
5952 return Register();
5953}
5954
5955Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5956 if ((Subtarget->hasFPARMv8())) {
5957 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5958 }
5959 return Register();
5960}
5961
5962Register fastEmit_ISD_UINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5963switch (RetVT.SimpleTy) {
5964 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5965 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5966 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5967 default: return Register();
5968}
5969}
5970
5971Register fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5972 if (RetVT.SimpleTy != MVT::v4f16)
5973 return Register();
5974 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5975 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5976 }
5977 return Register();
5978}
5979
5980Register fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5981 if (RetVT.SimpleTy != MVT::v8f16)
5982 return Register();
5983 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5984 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5985 }
5986 return Register();
5987}
5988
5989Register fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5990 if (RetVT.SimpleTy != MVT::v2f32)
5991 return Register();
5992 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5993 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5994 }
5995 return Register();
5996}
5997
5998Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5999 if (RetVT.SimpleTy != MVT::v4f32)
6000 return Register();
6001 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
6002 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
6003 }
6004 return Register();
6005}
6006
6007Register fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
6008 if (RetVT.SimpleTy != MVT::v2f64)
6009 return Register();
6010 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
6011 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
6012 }
6013 return Register();
6014}
6015
6016Register fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
6017 switch (VT.SimpleTy) {
6018 case MVT::i32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_r(RetVT, Op0);
6019 case MVT::i64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_r(RetVT, Op0);
6020 case MVT::v4i16: return fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
6021 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
6022 case MVT::v2i32: return fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
6023 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
6024 case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
6025 default: return Register();
6026 }
6027}
6028
6029// FastEmit functions for ISD::VECREDUCE_ADD.
6030
6031Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i8_r(MVT RetVT, Register Op0) {
6032 if (RetVT.SimpleTy != MVT::i8)
6033 return Register();
6034 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6035}
6036
6037Register fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(MVT RetVT, Register Op0) {
6038 if (RetVT.SimpleTy != MVT::i8)
6039 return Register();
6040 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6041}
6042
6043Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i16_r(MVT RetVT, Register Op0) {
6044 if (RetVT.SimpleTy != MVT::i16)
6045 return Register();
6046 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6047}
6048
6049Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(MVT RetVT, Register Op0) {
6050 if (RetVT.SimpleTy != MVT::i16)
6051 return Register();
6052 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6053}
6054
6055Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(MVT RetVT, Register Op0) {
6056 if (RetVT.SimpleTy != MVT::i32)
6057 return Register();
6058 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6059}
6060
6061Register fastEmit_ISD_VECREDUCE_ADD_MVT_v2i64_r(MVT RetVT, Register Op0) {
6062 if (RetVT.SimpleTy != MVT::i64)
6063 return Register();
6064 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6065}
6066
6067Register fastEmit_ISD_VECREDUCE_ADD_r(MVT VT, MVT RetVT, Register Op0) {
6068 switch (VT.SimpleTy) {
6069 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i8_r(RetVT, Op0);
6070 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(RetVT, Op0);
6071 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i16_r(RetVT, Op0);
6072 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(RetVT, Op0);
6073 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(RetVT, Op0);
6074 case MVT::v2i64: return fastEmit_ISD_VECREDUCE_ADD_MVT_v2i64_r(RetVT, Op0);
6075 default: return Register();
6076 }
6077}
6078
6079// FastEmit functions for ISD::VECREDUCE_FADD.
6080
6081Register fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(MVT RetVT, Register Op0) {
6082 if (RetVT.SimpleTy != MVT::f32)
6083 return Register();
6084 return fastEmitInst_r(MachineInstOpcode: AArch64::FADDPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6085}
6086
6087Register fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(MVT RetVT, Register Op0) {
6088 if (RetVT.SimpleTy != MVT::f64)
6089 return Register();
6090 return fastEmitInst_r(MachineInstOpcode: AArch64::FADDPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6091}
6092
6093Register fastEmit_ISD_VECREDUCE_FADD_r(MVT VT, MVT RetVT, Register Op0) {
6094 switch (VT.SimpleTy) {
6095 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(RetVT, Op0);
6096 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(RetVT, Op0);
6097 default: return Register();
6098 }
6099}
6100
6101// FastEmit functions for ISD::VECREDUCE_FMAX.
6102
6103Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f16_r(MVT RetVT, Register Op0) {
6104 if (RetVT.SimpleTy != MVT::f16)
6105 return Register();
6106 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6107 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6108 }
6109 return Register();
6110}
6111
6112Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v8f16_r(MVT RetVT, Register Op0) {
6113 if (RetVT.SimpleTy != MVT::f16)
6114 return Register();
6115 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6116 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6117 }
6118 return Register();
6119}
6120
6121Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f32_r(MVT RetVT, Register Op0) {
6122 if (RetVT.SimpleTy != MVT::f32)
6123 return Register();
6124 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6125}
6126
6127Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f32_r(MVT RetVT, Register Op0) {
6128 if (RetVT.SimpleTy != MVT::f32)
6129 return Register();
6130 if ((Subtarget->isNeonAvailable())) {
6131 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6132 }
6133 return Register();
6134}
6135
6136Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f64_r(MVT RetVT, Register Op0) {
6137 if (RetVT.SimpleTy != MVT::f64)
6138 return Register();
6139 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6140}
6141
6142Register fastEmit_ISD_VECREDUCE_FMAX_r(MVT VT, MVT RetVT, Register Op0) {
6143 switch (VT.SimpleTy) {
6144 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f16_r(RetVT, Op0);
6145 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v8f16_r(RetVT, Op0);
6146 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f32_r(RetVT, Op0);
6147 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f32_r(RetVT, Op0);
6148 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f64_r(RetVT, Op0);
6149 default: return Register();
6150 }
6151}
6152
6153// FastEmit functions for ISD::VECREDUCE_FMAXIMUM.
6154
6155Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f16_r(MVT RetVT, Register Op0) {
6156 if (RetVT.SimpleTy != MVT::f16)
6157 return Register();
6158 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6159 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6160 }
6161 return Register();
6162}
6163
6164Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v8f16_r(MVT RetVT, Register Op0) {
6165 if (RetVT.SimpleTy != MVT::f16)
6166 return Register();
6167 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6168 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6169 }
6170 return Register();
6171}
6172
6173Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f32_r(MVT RetVT, Register Op0) {
6174 if (RetVT.SimpleTy != MVT::f32)
6175 return Register();
6176 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6177}
6178
6179Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f32_r(MVT RetVT, Register Op0) {
6180 if (RetVT.SimpleTy != MVT::f32)
6181 return Register();
6182 if ((Subtarget->isNeonAvailable())) {
6183 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6184 }
6185 return Register();
6186}
6187
6188Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f64_r(MVT RetVT, Register Op0) {
6189 if (RetVT.SimpleTy != MVT::f64)
6190 return Register();
6191 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6192}
6193
6194Register fastEmit_ISD_VECREDUCE_FMAXIMUM_r(MVT VT, MVT RetVT, Register Op0) {
6195 switch (VT.SimpleTy) {
6196 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f16_r(RetVT, Op0);
6197 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v8f16_r(RetVT, Op0);
6198 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f32_r(RetVT, Op0);
6199 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f32_r(RetVT, Op0);
6200 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f64_r(RetVT, Op0);
6201 default: return Register();
6202 }
6203}
6204
6205// FastEmit functions for ISD::VECREDUCE_FMIN.
6206
6207Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f16_r(MVT RetVT, Register Op0) {
6208 if (RetVT.SimpleTy != MVT::f16)
6209 return Register();
6210 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6211 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6212 }
6213 return Register();
6214}
6215
6216Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v8f16_r(MVT RetVT, Register Op0) {
6217 if (RetVT.SimpleTy != MVT::f16)
6218 return Register();
6219 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6220 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6221 }
6222 return Register();
6223}
6224
6225Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f32_r(MVT RetVT, Register Op0) {
6226 if (RetVT.SimpleTy != MVT::f32)
6227 return Register();
6228 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6229}
6230
6231Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f32_r(MVT RetVT, Register Op0) {
6232 if (RetVT.SimpleTy != MVT::f32)
6233 return Register();
6234 if ((Subtarget->isNeonAvailable())) {
6235 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6236 }
6237 return Register();
6238}
6239
6240Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f64_r(MVT RetVT, Register Op0) {
6241 if (RetVT.SimpleTy != MVT::f64)
6242 return Register();
6243 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6244}
6245
6246Register fastEmit_ISD_VECREDUCE_FMIN_r(MVT VT, MVT RetVT, Register Op0) {
6247 switch (VT.SimpleTy) {
6248 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f16_r(RetVT, Op0);
6249 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v8f16_r(RetVT, Op0);
6250 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f32_r(RetVT, Op0);
6251 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f32_r(RetVT, Op0);
6252 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f64_r(RetVT, Op0);
6253 default: return Register();
6254 }
6255}
6256
6257// FastEmit functions for ISD::VECREDUCE_FMINIMUM.
6258
6259Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f16_r(MVT RetVT, Register Op0) {
6260 if (RetVT.SimpleTy != MVT::f16)
6261 return Register();
6262 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6263 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6264 }
6265 return Register();
6266}
6267
6268Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v8f16_r(MVT RetVT, Register Op0) {
6269 if (RetVT.SimpleTy != MVT::f16)
6270 return Register();
6271 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6272 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6273 }
6274 return Register();
6275}
6276
6277Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f32_r(MVT RetVT, Register Op0) {
6278 if (RetVT.SimpleTy != MVT::f32)
6279 return Register();
6280 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6281}
6282
6283Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f32_r(MVT RetVT, Register Op0) {
6284 if (RetVT.SimpleTy != MVT::f32)
6285 return Register();
6286 if ((Subtarget->isNeonAvailable())) {
6287 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6288 }
6289 return Register();
6290}
6291
6292Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f64_r(MVT RetVT, Register Op0) {
6293 if (RetVT.SimpleTy != MVT::f64)
6294 return Register();
6295 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6296}
6297
6298Register fastEmit_ISD_VECREDUCE_FMINIMUM_r(MVT VT, MVT RetVT, Register Op0) {
6299 switch (VT.SimpleTy) {
6300 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f16_r(RetVT, Op0);
6301 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v8f16_r(RetVT, Op0);
6302 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f32_r(RetVT, Op0);
6303 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f32_r(RetVT, Op0);
6304 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f64_r(RetVT, Op0);
6305 default: return Register();
6306 }
6307}
6308
6309// FastEmit functions for ISD::VECREDUCE_SMAX.
6310
6311Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i8_r(MVT RetVT, Register Op0) {
6312 if (RetVT.SimpleTy != MVT::i8)
6313 return Register();
6314 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6315}
6316
6317Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v16i8_r(MVT RetVT, Register Op0) {
6318 if (RetVT.SimpleTy != MVT::i8)
6319 return Register();
6320 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6321}
6322
6323Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i16_r(MVT RetVT, Register Op0) {
6324 if (RetVT.SimpleTy != MVT::i16)
6325 return Register();
6326 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6327}
6328
6329Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i16_r(MVT RetVT, Register Op0) {
6330 if (RetVT.SimpleTy != MVT::i16)
6331 return Register();
6332 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6333}
6334
6335Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i32_r(MVT RetVT, Register Op0) {
6336 if (RetVT.SimpleTy != MVT::i32)
6337 return Register();
6338 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6339}
6340
6341Register fastEmit_ISD_VECREDUCE_SMAX_r(MVT VT, MVT RetVT, Register Op0) {
6342 switch (VT.SimpleTy) {
6343 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i8_r(RetVT, Op0);
6344 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v16i8_r(RetVT, Op0);
6345 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i16_r(RetVT, Op0);
6346 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i16_r(RetVT, Op0);
6347 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i32_r(RetVT, Op0);
6348 default: return Register();
6349 }
6350}
6351
6352// FastEmit functions for ISD::VECREDUCE_SMIN.
6353
6354Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i8_r(MVT RetVT, Register Op0) {
6355 if (RetVT.SimpleTy != MVT::i8)
6356 return Register();
6357 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6358}
6359
6360Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v16i8_r(MVT RetVT, Register Op0) {
6361 if (RetVT.SimpleTy != MVT::i8)
6362 return Register();
6363 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6364}
6365
6366Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i16_r(MVT RetVT, Register Op0) {
6367 if (RetVT.SimpleTy != MVT::i16)
6368 return Register();
6369 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6370}
6371
6372Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i16_r(MVT RetVT, Register Op0) {
6373 if (RetVT.SimpleTy != MVT::i16)
6374 return Register();
6375 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6376}
6377
6378Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i32_r(MVT RetVT, Register Op0) {
6379 if (RetVT.SimpleTy != MVT::i32)
6380 return Register();
6381 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6382}
6383
6384Register fastEmit_ISD_VECREDUCE_SMIN_r(MVT VT, MVT RetVT, Register Op0) {
6385 switch (VT.SimpleTy) {
6386 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i8_r(RetVT, Op0);
6387 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v16i8_r(RetVT, Op0);
6388 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i16_r(RetVT, Op0);
6389 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i16_r(RetVT, Op0);
6390 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i32_r(RetVT, Op0);
6391 default: return Register();
6392 }
6393}
6394
6395// FastEmit functions for ISD::VECREDUCE_UMAX.
6396
6397Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i8_r(MVT RetVT, Register Op0) {
6398 if (RetVT.SimpleTy != MVT::i8)
6399 return Register();
6400 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6401}
6402
6403Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v16i8_r(MVT RetVT, Register Op0) {
6404 if (RetVT.SimpleTy != MVT::i8)
6405 return Register();
6406 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6407}
6408
6409Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i16_r(MVT RetVT, Register Op0) {
6410 if (RetVT.SimpleTy != MVT::i16)
6411 return Register();
6412 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6413}
6414
6415Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i16_r(MVT RetVT, Register Op0) {
6416 if (RetVT.SimpleTy != MVT::i16)
6417 return Register();
6418 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6419}
6420
6421Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i32_r(MVT RetVT, Register Op0) {
6422 if (RetVT.SimpleTy != MVT::i32)
6423 return Register();
6424 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6425}
6426
6427Register fastEmit_ISD_VECREDUCE_UMAX_r(MVT VT, MVT RetVT, Register Op0) {
6428 switch (VT.SimpleTy) {
6429 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i8_r(RetVT, Op0);
6430 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v16i8_r(RetVT, Op0);
6431 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i16_r(RetVT, Op0);
6432 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i16_r(RetVT, Op0);
6433 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i32_r(RetVT, Op0);
6434 default: return Register();
6435 }
6436}
6437
6438// FastEmit functions for ISD::VECREDUCE_UMIN.
6439
6440Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i8_r(MVT RetVT, Register Op0) {
6441 if (RetVT.SimpleTy != MVT::i8)
6442 return Register();
6443 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6444}
6445
6446Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v16i8_r(MVT RetVT, Register Op0) {
6447 if (RetVT.SimpleTy != MVT::i8)
6448 return Register();
6449 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6450}
6451
6452Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i16_r(MVT RetVT, Register Op0) {
6453 if (RetVT.SimpleTy != MVT::i16)
6454 return Register();
6455 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6456}
6457
6458Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i16_r(MVT RetVT, Register Op0) {
6459 if (RetVT.SimpleTy != MVT::i16)
6460 return Register();
6461 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6462}
6463
6464Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i32_r(MVT RetVT, Register Op0) {
6465 if (RetVT.SimpleTy != MVT::i32)
6466 return Register();
6467 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6468}
6469
6470Register fastEmit_ISD_VECREDUCE_UMIN_r(MVT VT, MVT RetVT, Register Op0) {
6471 switch (VT.SimpleTy) {
6472 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i8_r(RetVT, Op0);
6473 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v16i8_r(RetVT, Op0);
6474 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i16_r(RetVT, Op0);
6475 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i16_r(RetVT, Op0);
6476 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i32_r(RetVT, Op0);
6477 default: return Register();
6478 }
6479}
6480
6481// FastEmit functions for ISD::VECTOR_REVERSE.
6482
6483Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(MVT RetVT, Register Op0) {
6484 if (RetVT.SimpleTy != MVT::nxv2i1)
6485 return Register();
6486 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6487 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_D, RC: &AArch64::PPRRegClass, Op0);
6488 }
6489 return Register();
6490}
6491
6492Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(MVT RetVT, Register Op0) {
6493 if (RetVT.SimpleTy != MVT::nxv4i1)
6494 return Register();
6495 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6496 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_S, RC: &AArch64::PPRRegClass, Op0);
6497 }
6498 return Register();
6499}
6500
6501Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(MVT RetVT, Register Op0) {
6502 if (RetVT.SimpleTy != MVT::nxv8i1)
6503 return Register();
6504 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6505 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_H, RC: &AArch64::PPRRegClass, Op0);
6506 }
6507 return Register();
6508}
6509
6510Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(MVT RetVT, Register Op0) {
6511 if (RetVT.SimpleTy != MVT::nxv16i1)
6512 return Register();
6513 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6514 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_B, RC: &AArch64::PPRRegClass, Op0);
6515 }
6516 return Register();
6517}
6518
6519Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
6520 if (RetVT.SimpleTy != MVT::nxv16i8)
6521 return Register();
6522 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6523 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_B, RC: &AArch64::ZPRRegClass, Op0);
6524 }
6525 return Register();
6526}
6527
6528Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
6529 if (RetVT.SimpleTy != MVT::nxv8i16)
6530 return Register();
6531 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6532 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6533 }
6534 return Register();
6535}
6536
6537Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
6538 if (RetVT.SimpleTy != MVT::nxv4i32)
6539 return Register();
6540 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6541 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6542 }
6543 return Register();
6544}
6545
6546Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(MVT RetVT, Register Op0) {
6547 if (RetVT.SimpleTy != MVT::nxv2i64)
6548 return Register();
6549 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6550 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6551 }
6552 return Register();
6553}
6554
6555Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(MVT RetVT, Register Op0) {
6556 if (RetVT.SimpleTy != MVT::nxv2f16)
6557 return Register();
6558 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6559 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6560 }
6561 return Register();
6562}
6563
6564Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(MVT RetVT, Register Op0) {
6565 if (RetVT.SimpleTy != MVT::nxv4f16)
6566 return Register();
6567 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6568 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6569 }
6570 return Register();
6571}
6572
6573Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
6574 if (RetVT.SimpleTy != MVT::nxv8f16)
6575 return Register();
6576 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6577 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6578 }
6579 return Register();
6580}
6581
6582Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(MVT RetVT, Register Op0) {
6583 if (RetVT.SimpleTy != MVT::nxv2bf16)
6584 return Register();
6585 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6586 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6587 }
6588 return Register();
6589}
6590
6591Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(MVT RetVT, Register Op0) {
6592 if (RetVT.SimpleTy != MVT::nxv4bf16)
6593 return Register();
6594 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6595 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6596 }
6597 return Register();
6598}
6599
6600Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(MVT RetVT, Register Op0) {
6601 if (RetVT.SimpleTy != MVT::nxv8bf16)
6602 return Register();
6603 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6604 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6605 }
6606 return Register();
6607}
6608
6609Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(MVT RetVT, Register Op0) {
6610 if (RetVT.SimpleTy != MVT::nxv2f32)
6611 return Register();
6612 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6613 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6614 }
6615 return Register();
6616}
6617
6618Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
6619 if (RetVT.SimpleTy != MVT::nxv4f32)
6620 return Register();
6621 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6622 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6623 }
6624 return Register();
6625}
6626
6627Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
6628 if (RetVT.SimpleTy != MVT::nxv2f64)
6629 return Register();
6630 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6631 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6632 }
6633 return Register();
6634}
6635
6636Register fastEmit_ISD_VECTOR_REVERSE_r(MVT VT, MVT RetVT, Register Op0) {
6637 switch (VT.SimpleTy) {
6638 case MVT::nxv2i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(RetVT, Op0);
6639 case MVT::nxv4i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(RetVT, Op0);
6640 case MVT::nxv8i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(RetVT, Op0);
6641 case MVT::nxv16i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(RetVT, Op0);
6642 case MVT::nxv16i8: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(RetVT, Op0);
6643 case MVT::nxv8i16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(RetVT, Op0);
6644 case MVT::nxv4i32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(RetVT, Op0);
6645 case MVT::nxv2i64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(RetVT, Op0);
6646 case MVT::nxv2f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(RetVT, Op0);
6647 case MVT::nxv4f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(RetVT, Op0);
6648 case MVT::nxv8f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(RetVT, Op0);
6649 case MVT::nxv2bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(RetVT, Op0);
6650 case MVT::nxv4bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(RetVT, Op0);
6651 case MVT::nxv8bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(RetVT, Op0);
6652 case MVT::nxv2f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(RetVT, Op0);
6653 case MVT::nxv4f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(RetVT, Op0);
6654 case MVT::nxv2f64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(RetVT, Op0);
6655 default: return Register();
6656 }
6657}
6658
6659// Top-level FastEmit function.
6660
6661Register fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, Register Op0) override {
6662 switch (Opcode) {
6663 case AArch64ISD::COALESCER_BARRIER: return fastEmit_AArch64ISD_COALESCER_BARRIER_r(VT, RetVT, Op0);
6664 case AArch64ISD::DUP: return fastEmit_AArch64ISD_DUP_r(VT, RetVT, Op0);
6665 case AArch64ISD::FCVTXN: return fastEmit_AArch64ISD_FCVTXN_r(VT, RetVT, Op0);
6666 case AArch64ISD::FRECPE: return fastEmit_AArch64ISD_FRECPE_r(VT, RetVT, Op0);
6667 case AArch64ISD::FRSQRTE: return fastEmit_AArch64ISD_FRSQRTE_r(VT, RetVT, Op0);
6668 case AArch64ISD::REV16: return fastEmit_AArch64ISD_REV16_r(VT, RetVT, Op0);
6669 case AArch64ISD::REV32: return fastEmit_AArch64ISD_REV32_r(VT, RetVT, Op0);
6670 case AArch64ISD::REV64: return fastEmit_AArch64ISD_REV64_r(VT, RetVT, Op0);
6671 case AArch64ISD::SADDLP: return fastEmit_AArch64ISD_SADDLP_r(VT, RetVT, Op0);
6672 case AArch64ISD::SITOF: return fastEmit_AArch64ISD_SITOF_r(VT, RetVT, Op0);
6673 case AArch64ISD::SQABS: return fastEmit_AArch64ISD_SQABS_r(VT, RetVT, Op0);
6674 case AArch64ISD::SQNEG: return fastEmit_AArch64ISD_SQNEG_r(VT, RetVT, Op0);
6675 case AArch64ISD::SUNPKHI: return fastEmit_AArch64ISD_SUNPKHI_r(VT, RetVT, Op0);
6676 case AArch64ISD::SUNPKLO: return fastEmit_AArch64ISD_SUNPKLO_r(VT, RetVT, Op0);
6677 case AArch64ISD::UADDLP: return fastEmit_AArch64ISD_UADDLP_r(VT, RetVT, Op0);
6678 case AArch64ISD::UITOF: return fastEmit_AArch64ISD_UITOF_r(VT, RetVT, Op0);
6679 case AArch64ISD::UUNPKHI: return fastEmit_AArch64ISD_UUNPKHI_r(VT, RetVT, Op0);
6680 case AArch64ISD::UUNPKLO: return fastEmit_AArch64ISD_UUNPKLO_r(VT, RetVT, Op0);
6681 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
6682 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
6683 case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0);
6684 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
6685 case ISD::CTLS: return fastEmit_ISD_CTLS_r(VT, RetVT, Op0);
6686 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
6687 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
6688 case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0);
6689 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
6690 case ISD::FCEIL: return fastEmit_ISD_FCEIL_r(VT, RetVT, Op0);
6691 case ISD::FFLOOR: return fastEmit_ISD_FFLOOR_r(VT, RetVT, Op0);
6692 case ISD::FNEARBYINT: return fastEmit_ISD_FNEARBYINT_r(VT, RetVT, Op0);
6693 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
6694 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
6695 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
6696 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
6697 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
6698 case ISD::FRINT: return fastEmit_ISD_FRINT_r(VT, RetVT, Op0);
6699 case ISD::FROUND: return fastEmit_ISD_FROUND_r(VT, RetVT, Op0);
6700 case ISD::FROUNDEVEN: return fastEmit_ISD_FROUNDEVEN_r(VT, RetVT, Op0);
6701 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
6702 case ISD::FTRUNC: return fastEmit_ISD_FTRUNC_r(VT, RetVT, Op0);
6703 case ISD::LLROUND: return fastEmit_ISD_LLROUND_r(VT, RetVT, Op0);
6704 case ISD::LROUND: return fastEmit_ISD_LROUND_r(VT, RetVT, Op0);
6705 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
6706 case ISD::SPLAT_VECTOR: return fastEmit_ISD_SPLAT_VECTOR_r(VT, RetVT, Op0);
6707 case ISD::STRICT_FCEIL: return fastEmit_ISD_STRICT_FCEIL_r(VT, RetVT, Op0);
6708 case ISD::STRICT_FFLOOR: return fastEmit_ISD_STRICT_FFLOOR_r(VT, RetVT, Op0);
6709 case ISD::STRICT_FNEARBYINT: return fastEmit_ISD_STRICT_FNEARBYINT_r(VT, RetVT, Op0);
6710 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0);
6711 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0);
6712 case ISD::STRICT_FP_TO_SINT: return fastEmit_ISD_STRICT_FP_TO_SINT_r(VT, RetVT, Op0);
6713 case ISD::STRICT_FP_TO_UINT: return fastEmit_ISD_STRICT_FP_TO_UINT_r(VT, RetVT, Op0);
6714 case ISD::STRICT_FRINT: return fastEmit_ISD_STRICT_FRINT_r(VT, RetVT, Op0);
6715 case ISD::STRICT_FROUND: return fastEmit_ISD_STRICT_FROUND_r(VT, RetVT, Op0);
6716 case ISD::STRICT_FROUNDEVEN: return fastEmit_ISD_STRICT_FROUNDEVEN_r(VT, RetVT, Op0);
6717 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0);
6718 case ISD::STRICT_FTRUNC: return fastEmit_ISD_STRICT_FTRUNC_r(VT, RetVT, Op0);
6719 case ISD::STRICT_LLROUND: return fastEmit_ISD_STRICT_LLROUND_r(VT, RetVT, Op0);
6720 case ISD::STRICT_LROUND: return fastEmit_ISD_STRICT_LROUND_r(VT, RetVT, Op0);
6721 case ISD::STRICT_SINT_TO_FP: return fastEmit_ISD_STRICT_SINT_TO_FP_r(VT, RetVT, Op0);
6722 case ISD::STRICT_UINT_TO_FP: return fastEmit_ISD_STRICT_UINT_TO_FP_r(VT, RetVT, Op0);
6723 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
6724 case ISD::TRUNCATE_SSAT_S: return fastEmit_ISD_TRUNCATE_SSAT_S_r(VT, RetVT, Op0);
6725 case ISD::TRUNCATE_SSAT_U: return fastEmit_ISD_TRUNCATE_SSAT_U_r(VT, RetVT, Op0);
6726 case ISD::TRUNCATE_USAT_U: return fastEmit_ISD_TRUNCATE_USAT_U_r(VT, RetVT, Op0);
6727 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
6728 case ISD::VECREDUCE_ADD: return fastEmit_ISD_VECREDUCE_ADD_r(VT, RetVT, Op0);
6729 case ISD::VECREDUCE_FADD: return fastEmit_ISD_VECREDUCE_FADD_r(VT, RetVT, Op0);
6730 case ISD::VECREDUCE_FMAX: return fastEmit_ISD_VECREDUCE_FMAX_r(VT, RetVT, Op0);
6731 case ISD::VECREDUCE_FMAXIMUM: return fastEmit_ISD_VECREDUCE_FMAXIMUM_r(VT, RetVT, Op0);
6732 case ISD::VECREDUCE_FMIN: return fastEmit_ISD_VECREDUCE_FMIN_r(VT, RetVT, Op0);
6733 case ISD::VECREDUCE_FMINIMUM: return fastEmit_ISD_VECREDUCE_FMINIMUM_r(VT, RetVT, Op0);
6734 case ISD::VECREDUCE_SMAX: return fastEmit_ISD_VECREDUCE_SMAX_r(VT, RetVT, Op0);
6735 case ISD::VECREDUCE_SMIN: return fastEmit_ISD_VECREDUCE_SMIN_r(VT, RetVT, Op0);
6736 case ISD::VECREDUCE_UMAX: return fastEmit_ISD_VECREDUCE_UMAX_r(VT, RetVT, Op0);
6737 case ISD::VECREDUCE_UMIN: return fastEmit_ISD_VECREDUCE_UMIN_r(VT, RetVT, Op0);
6738 case ISD::VECTOR_REVERSE: return fastEmit_ISD_VECTOR_REVERSE_r(VT, RetVT, Op0);
6739 default: return Register();
6740 }
6741}
6742
6743// FastEmit functions for AArch64ISD::ADDHN.
6744
6745Register fastEmit_AArch64ISD_ADDHN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6746 if (RetVT.SimpleTy != MVT::v8i8)
6747 return Register();
6748 if ((Subtarget->isNeonAvailable())) {
6749 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDHNv8i16_v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
6750 }
6751 return Register();
6752}
6753
6754Register fastEmit_AArch64ISD_ADDHN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6755 if (RetVT.SimpleTy != MVT::v4i16)
6756 return Register();
6757 if ((Subtarget->isNeonAvailable())) {
6758 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDHNv4i32_v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6759 }
6760 return Register();
6761}
6762
6763Register fastEmit_AArch64ISD_ADDHN_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6764 if (RetVT.SimpleTy != MVT::v2i32)
6765 return Register();
6766 if ((Subtarget->isNeonAvailable())) {
6767 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDHNv2i64_v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6768 }
6769 return Register();
6770}
6771
6772Register fastEmit_AArch64ISD_ADDHN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6773 switch (VT.SimpleTy) {
6774 case MVT::v8i16: return fastEmit_AArch64ISD_ADDHN_MVT_v8i16_rr(RetVT, Op0, Op1);
6775 case MVT::v4i32: return fastEmit_AArch64ISD_ADDHN_MVT_v4i32_rr(RetVT, Op0, Op1);
6776 case MVT::v2i64: return fastEmit_AArch64ISD_ADDHN_MVT_v2i64_rr(RetVT, Op0, Op1);
6777 default: return Register();
6778 }
6779}
6780
6781// FastEmit functions for AArch64ISD::ADDP.
6782
6783Register fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6784 if (RetVT.SimpleTy != MVT::v8i8)
6785 return Register();
6786 if ((Subtarget->isNeonAvailable())) {
6787 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
6788 }
6789 return Register();
6790}
6791
6792Register fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6793 if (RetVT.SimpleTy != MVT::v16i8)
6794 return Register();
6795 if ((Subtarget->isNeonAvailable())) {
6796 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
6797 }
6798 return Register();
6799}
6800
6801Register fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6802 if (RetVT.SimpleTy != MVT::v4i16)
6803 return Register();
6804 if ((Subtarget->isNeonAvailable())) {
6805 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6806 }
6807 return Register();
6808}
6809
6810Register fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6811 if (RetVT.SimpleTy != MVT::v8i16)
6812 return Register();
6813 if ((Subtarget->isNeonAvailable())) {
6814 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
6815 }
6816 return Register();
6817}
6818
6819Register fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6820 if (RetVT.SimpleTy != MVT::v2i32)
6821 return Register();
6822 if ((Subtarget->isNeonAvailable())) {
6823 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6824 }
6825 return Register();
6826}
6827
6828Register fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6829 if (RetVT.SimpleTy != MVT::v4i32)
6830 return Register();
6831 if ((Subtarget->isNeonAvailable())) {
6832 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
6833 }
6834 return Register();
6835}
6836
6837Register fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6838 if (RetVT.SimpleTy != MVT::v2i64)
6839 return Register();
6840 if ((Subtarget->isNeonAvailable())) {
6841 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
6842 }
6843 return Register();
6844}
6845
6846Register fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
6847 if (RetVT.SimpleTy != MVT::v4f16)
6848 return Register();
6849 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6850 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6851 }
6852 return Register();
6853}
6854
6855Register fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
6856 if (RetVT.SimpleTy != MVT::v8f16)
6857 return Register();
6858 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6859 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
6860 }
6861 return Register();
6862}
6863
6864Register fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
6865 if (RetVT.SimpleTy != MVT::v2f32)
6866 return Register();
6867 if ((Subtarget->isNeonAvailable())) {
6868 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6869 }
6870 return Register();
6871}
6872
6873Register fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
6874 if (RetVT.SimpleTy != MVT::v4f32)
6875 return Register();
6876 if ((Subtarget->isNeonAvailable())) {
6877 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
6878 }
6879 return Register();
6880}
6881
6882Register fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
6883 if (RetVT.SimpleTy != MVT::v2f64)
6884 return Register();
6885 if ((Subtarget->isNeonAvailable())) {
6886 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
6887 }
6888 return Register();
6889}
6890
6891Register fastEmit_AArch64ISD_ADDP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6892 switch (VT.SimpleTy) {
6893 case MVT::v8i8: return fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(RetVT, Op0, Op1);
6894 case MVT::v16i8: return fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(RetVT, Op0, Op1);
6895 case MVT::v4i16: return fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(RetVT, Op0, Op1);
6896 case MVT::v8i16: return fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(RetVT, Op0, Op1);
6897 case MVT::v2i32: return fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(RetVT, Op0, Op1);
6898 case MVT::v4i32: return fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(RetVT, Op0, Op1);
6899 case MVT::v2i64: return fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(RetVT, Op0, Op1);
6900 case MVT::v4f16: return fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(RetVT, Op0, Op1);
6901 case MVT::v8f16: return fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(RetVT, Op0, Op1);
6902 case MVT::v2f32: return fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(RetVT, Op0, Op1);
6903 case MVT::v4f32: return fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(RetVT, Op0, Op1);
6904 case MVT::v2f64: return fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(RetVT, Op0, Op1);
6905 default: return Register();
6906 }
6907}
6908
6909// FastEmit functions for AArch64ISD::BIC.
6910
6911Register fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6912 if (RetVT.SimpleTy != MVT::nxv16i8)
6913 return Register();
6914 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6915 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6916 }
6917 return Register();
6918}
6919
6920Register fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6921 if (RetVT.SimpleTy != MVT::nxv8i16)
6922 return Register();
6923 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6924 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6925 }
6926 return Register();
6927}
6928
6929Register fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6930 if (RetVT.SimpleTy != MVT::nxv4i32)
6931 return Register();
6932 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6933 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6934 }
6935 return Register();
6936}
6937
6938Register fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6939 if (RetVT.SimpleTy != MVT::nxv2i64)
6940 return Register();
6941 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6942 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6943 }
6944 return Register();
6945}
6946
6947Register fastEmit_AArch64ISD_BIC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6948 switch (VT.SimpleTy) {
6949 case MVT::nxv16i8: return fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(RetVT, Op0, Op1);
6950 case MVT::nxv8i16: return fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(RetVT, Op0, Op1);
6951 case MVT::nxv4i32: return fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(RetVT, Op0, Op1);
6952 case MVT::nxv2i64: return fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(RetVT, Op0, Op1);
6953 default: return Register();
6954 }
6955}
6956
6957// FastEmit functions for AArch64ISD::CMTST.
6958
6959Register fastEmit_AArch64ISD_CMTST_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6960 if (RetVT.SimpleTy != MVT::v8i8)
6961 return Register();
6962 if ((Subtarget->isNeonAvailable())) {
6963 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
6964 }
6965 return Register();
6966}
6967
6968Register fastEmit_AArch64ISD_CMTST_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6969 if (RetVT.SimpleTy != MVT::v16i8)
6970 return Register();
6971 if ((Subtarget->isNeonAvailable())) {
6972 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
6973 }
6974 return Register();
6975}
6976
6977Register fastEmit_AArch64ISD_CMTST_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6978 if (RetVT.SimpleTy != MVT::v4i16)
6979 return Register();
6980 if ((Subtarget->isNeonAvailable())) {
6981 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6982 }
6983 return Register();
6984}
6985
6986Register fastEmit_AArch64ISD_CMTST_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6987 if (RetVT.SimpleTy != MVT::v8i16)
6988 return Register();
6989 if ((Subtarget->isNeonAvailable())) {
6990 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
6991 }
6992 return Register();
6993}
6994
6995Register fastEmit_AArch64ISD_CMTST_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6996 if (RetVT.SimpleTy != MVT::v2i32)
6997 return Register();
6998 if ((Subtarget->isNeonAvailable())) {
6999 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7000 }
7001 return Register();
7002}
7003
7004Register fastEmit_AArch64ISD_CMTST_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7005 if (RetVT.SimpleTy != MVT::v4i32)
7006 return Register();
7007 if ((Subtarget->isNeonAvailable())) {
7008 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7009 }
7010 return Register();
7011}
7012
7013Register fastEmit_AArch64ISD_CMTST_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7014 if (RetVT.SimpleTy != MVT::v1i64)
7015 return Register();
7016 if ((Subtarget->isNeonAvailable())) {
7017 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7018 }
7019 return Register();
7020}
7021
7022Register fastEmit_AArch64ISD_CMTST_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7023 if (RetVT.SimpleTy != MVT::v2i64)
7024 return Register();
7025 if ((Subtarget->isNeonAvailable())) {
7026 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7027 }
7028 return Register();
7029}
7030
7031Register fastEmit_AArch64ISD_CMTST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7032 switch (VT.SimpleTy) {
7033 case MVT::v8i8: return fastEmit_AArch64ISD_CMTST_MVT_v8i8_rr(RetVT, Op0, Op1);
7034 case MVT::v16i8: return fastEmit_AArch64ISD_CMTST_MVT_v16i8_rr(RetVT, Op0, Op1);
7035 case MVT::v4i16: return fastEmit_AArch64ISD_CMTST_MVT_v4i16_rr(RetVT, Op0, Op1);
7036 case MVT::v8i16: return fastEmit_AArch64ISD_CMTST_MVT_v8i16_rr(RetVT, Op0, Op1);
7037 case MVT::v2i32: return fastEmit_AArch64ISD_CMTST_MVT_v2i32_rr(RetVT, Op0, Op1);
7038 case MVT::v4i32: return fastEmit_AArch64ISD_CMTST_MVT_v4i32_rr(RetVT, Op0, Op1);
7039 case MVT::v1i64: return fastEmit_AArch64ISD_CMTST_MVT_v1i64_rr(RetVT, Op0, Op1);
7040 case MVT::v2i64: return fastEmit_AArch64ISD_CMTST_MVT_v2i64_rr(RetVT, Op0, Op1);
7041 default: return Register();
7042 }
7043}
7044
7045// FastEmit functions for AArch64ISD::CTTZ_ELTS_ZERO_POISON.
7046
7047Register fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
7048 if (RetVT.SimpleTy != MVT::i64)
7049 return Register();
7050 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()))) {
7051 return fastEmitInst_rr(MachineInstOpcode: AArch64::FIRSTP_XPP_D, RC: &AArch64::GPR64RegClass, Op0, Op1);
7052 }
7053 return Register();
7054}
7055
7056Register fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
7057 if (RetVT.SimpleTy != MVT::i64)
7058 return Register();
7059 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()))) {
7060 return fastEmitInst_rr(MachineInstOpcode: AArch64::FIRSTP_XPP_S, RC: &AArch64::GPR64RegClass, Op0, Op1);
7061 }
7062 return Register();
7063}
7064
7065Register fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
7066 if (RetVT.SimpleTy != MVT::i64)
7067 return Register();
7068 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()))) {
7069 return fastEmitInst_rr(MachineInstOpcode: AArch64::FIRSTP_XPP_H, RC: &AArch64::GPR64RegClass, Op0, Op1);
7070 }
7071 return Register();
7072}
7073
7074Register fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7075 if (RetVT.SimpleTy != MVT::i64)
7076 return Register();
7077 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()))) {
7078 return fastEmitInst_rr(MachineInstOpcode: AArch64::FIRSTP_XPP_B, RC: &AArch64::GPR64RegClass, Op0, Op1);
7079 }
7080 return Register();
7081}
7082
7083Register fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7084 switch (VT.SimpleTy) {
7085 case MVT::nxv2i1: return fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv2i1_rr(RetVT, Op0, Op1);
7086 case MVT::nxv4i1: return fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv4i1_rr(RetVT, Op0, Op1);
7087 case MVT::nxv8i1: return fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv8i1_rr(RetVT, Op0, Op1);
7088 case MVT::nxv16i1: return fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7089 default: return Register();
7090 }
7091}
7092
7093// FastEmit functions for AArch64ISD::FCMEQ.
7094
7095Register fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7096 if (RetVT.SimpleTy != MVT::i32)
7097 return Register();
7098 if ((Subtarget->isNeonAvailable())) {
7099 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7100 }
7101 return Register();
7102}
7103
7104Register fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7105 if (RetVT.SimpleTy != MVT::i64)
7106 return Register();
7107 if ((Subtarget->isNeonAvailable())) {
7108 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7109 }
7110 return Register();
7111}
7112
7113Register fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7114 if (RetVT.SimpleTy != MVT::v4i16)
7115 return Register();
7116 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7117 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7118 }
7119 return Register();
7120}
7121
7122Register fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7123 if (RetVT.SimpleTy != MVT::v8i16)
7124 return Register();
7125 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7126 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7127 }
7128 return Register();
7129}
7130
7131Register fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7132 if (RetVT.SimpleTy != MVT::v2i32)
7133 return Register();
7134 if ((Subtarget->isNeonAvailable())) {
7135 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7136 }
7137 return Register();
7138}
7139
7140Register fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7141 if (RetVT.SimpleTy != MVT::v4i32)
7142 return Register();
7143 if ((Subtarget->isNeonAvailable())) {
7144 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7145 }
7146 return Register();
7147}
7148
7149Register fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7150 if (RetVT.SimpleTy != MVT::v1i64)
7151 return Register();
7152 if ((Subtarget->isNeonAvailable())) {
7153 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7154 }
7155 return Register();
7156}
7157
7158Register fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7159 if (RetVT.SimpleTy != MVT::v2i64)
7160 return Register();
7161 if ((Subtarget->isNeonAvailable())) {
7162 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7163 }
7164 return Register();
7165}
7166
7167Register fastEmit_AArch64ISD_FCMEQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7168 switch (VT.SimpleTy) {
7169 case MVT::f32: return fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(RetVT, Op0, Op1);
7170 case MVT::f64: return fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(RetVT, Op0, Op1);
7171 case MVT::v4f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(RetVT, Op0, Op1);
7172 case MVT::v8f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(RetVT, Op0, Op1);
7173 case MVT::v2f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(RetVT, Op0, Op1);
7174 case MVT::v4f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(RetVT, Op0, Op1);
7175 case MVT::v1f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(RetVT, Op0, Op1);
7176 case MVT::v2f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(RetVT, Op0, Op1);
7177 default: return Register();
7178 }
7179}
7180
7181// FastEmit functions for AArch64ISD::FCMGE.
7182
7183Register fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7184 if (RetVT.SimpleTy != MVT::i32)
7185 return Register();
7186 if ((Subtarget->isNeonAvailable())) {
7187 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7188 }
7189 return Register();
7190}
7191
7192Register fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7193 if (RetVT.SimpleTy != MVT::i64)
7194 return Register();
7195 if ((Subtarget->isNeonAvailable())) {
7196 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7197 }
7198 return Register();
7199}
7200
7201Register fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7202 if (RetVT.SimpleTy != MVT::v4i16)
7203 return Register();
7204 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7205 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7206 }
7207 return Register();
7208}
7209
7210Register fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7211 if (RetVT.SimpleTy != MVT::v8i16)
7212 return Register();
7213 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7214 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7215 }
7216 return Register();
7217}
7218
7219Register fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7220 if (RetVT.SimpleTy != MVT::v2i32)
7221 return Register();
7222 if ((Subtarget->isNeonAvailable())) {
7223 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7224 }
7225 return Register();
7226}
7227
7228Register fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7229 if (RetVT.SimpleTy != MVT::v4i32)
7230 return Register();
7231 if ((Subtarget->isNeonAvailable())) {
7232 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7233 }
7234 return Register();
7235}
7236
7237Register fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7238 if (RetVT.SimpleTy != MVT::v1i64)
7239 return Register();
7240 if ((Subtarget->isNeonAvailable())) {
7241 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7242 }
7243 return Register();
7244}
7245
7246Register fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7247 if (RetVT.SimpleTy != MVT::v2i64)
7248 return Register();
7249 if ((Subtarget->isNeonAvailable())) {
7250 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7251 }
7252 return Register();
7253}
7254
7255Register fastEmit_AArch64ISD_FCMGE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7256 switch (VT.SimpleTy) {
7257 case MVT::f32: return fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(RetVT, Op0, Op1);
7258 case MVT::f64: return fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(RetVT, Op0, Op1);
7259 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(RetVT, Op0, Op1);
7260 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(RetVT, Op0, Op1);
7261 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(RetVT, Op0, Op1);
7262 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(RetVT, Op0, Op1);
7263 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(RetVT, Op0, Op1);
7264 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(RetVT, Op0, Op1);
7265 default: return Register();
7266 }
7267}
7268
7269// FastEmit functions for AArch64ISD::FCMGT.
7270
7271Register fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7272 if (RetVT.SimpleTy != MVT::i32)
7273 return Register();
7274 if ((Subtarget->isNeonAvailable())) {
7275 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7276 }
7277 return Register();
7278}
7279
7280Register fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7281 if (RetVT.SimpleTy != MVT::i64)
7282 return Register();
7283 if ((Subtarget->isNeonAvailable())) {
7284 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7285 }
7286 return Register();
7287}
7288
7289Register fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7290 if (RetVT.SimpleTy != MVT::v4i16)
7291 return Register();
7292 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7293 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7294 }
7295 return Register();
7296}
7297
7298Register fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7299 if (RetVT.SimpleTy != MVT::v8i16)
7300 return Register();
7301 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7302 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7303 }
7304 return Register();
7305}
7306
7307Register fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7308 if (RetVT.SimpleTy != MVT::v2i32)
7309 return Register();
7310 if ((Subtarget->isNeonAvailable())) {
7311 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7312 }
7313 return Register();
7314}
7315
7316Register fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7317 if (RetVT.SimpleTy != MVT::v4i32)
7318 return Register();
7319 if ((Subtarget->isNeonAvailable())) {
7320 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7321 }
7322 return Register();
7323}
7324
7325Register fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7326 if (RetVT.SimpleTy != MVT::v1i64)
7327 return Register();
7328 if ((Subtarget->isNeonAvailable())) {
7329 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7330 }
7331 return Register();
7332}
7333
7334Register fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7335 if (RetVT.SimpleTy != MVT::v2i64)
7336 return Register();
7337 if ((Subtarget->isNeonAvailable())) {
7338 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7339 }
7340 return Register();
7341}
7342
7343Register fastEmit_AArch64ISD_FCMGT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7344 switch (VT.SimpleTy) {
7345 case MVT::f32: return fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(RetVT, Op0, Op1);
7346 case MVT::f64: return fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(RetVT, Op0, Op1);
7347 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(RetVT, Op0, Op1);
7348 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(RetVT, Op0, Op1);
7349 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(RetVT, Op0, Op1);
7350 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(RetVT, Op0, Op1);
7351 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(RetVT, Op0, Op1);
7352 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(RetVT, Op0, Op1);
7353 default: return Register();
7354 }
7355}
7356
7357// FastEmit functions for AArch64ISD::FRECPS.
7358
7359Register fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7360 if (RetVT.SimpleTy != MVT::f32)
7361 return Register();
7362 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7363}
7364
7365Register fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7366 if (RetVT.SimpleTy != MVT::f64)
7367 return Register();
7368 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7369}
7370
7371Register fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7372 if (RetVT.SimpleTy != MVT::v2f32)
7373 return Register();
7374 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7375}
7376
7377Register fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7378 if (RetVT.SimpleTy != MVT::v4f32)
7379 return Register();
7380 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7381}
7382
7383Register fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7384 if (RetVT.SimpleTy != MVT::v2f64)
7385 return Register();
7386 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7387}
7388
7389Register fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7390 if (RetVT.SimpleTy != MVT::nxv8f16)
7391 return Register();
7392 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7393 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7394 }
7395 return Register();
7396}
7397
7398Register fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7399 if (RetVT.SimpleTy != MVT::nxv4f32)
7400 return Register();
7401 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7402 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7403 }
7404 return Register();
7405}
7406
7407Register fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7408 if (RetVT.SimpleTy != MVT::nxv2f64)
7409 return Register();
7410 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7411 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7412 }
7413 return Register();
7414}
7415
7416Register fastEmit_AArch64ISD_FRECPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7417 switch (VT.SimpleTy) {
7418 case MVT::f32: return fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(RetVT, Op0, Op1);
7419 case MVT::f64: return fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(RetVT, Op0, Op1);
7420 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(RetVT, Op0, Op1);
7421 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(RetVT, Op0, Op1);
7422 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(RetVT, Op0, Op1);
7423 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(RetVT, Op0, Op1);
7424 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(RetVT, Op0, Op1);
7425 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(RetVT, Op0, Op1);
7426 default: return Register();
7427 }
7428}
7429
7430// FastEmit functions for AArch64ISD::FRSQRTS.
7431
7432Register fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7433 if (RetVT.SimpleTy != MVT::f32)
7434 return Register();
7435 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7436}
7437
7438Register fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7439 if (RetVT.SimpleTy != MVT::f64)
7440 return Register();
7441 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7442}
7443
7444Register fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7445 if (RetVT.SimpleTy != MVT::v2f32)
7446 return Register();
7447 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7448}
7449
7450Register fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7451 if (RetVT.SimpleTy != MVT::v4f32)
7452 return Register();
7453 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7454}
7455
7456Register fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7457 if (RetVT.SimpleTy != MVT::v2f64)
7458 return Register();
7459 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7460}
7461
7462Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7463 if (RetVT.SimpleTy != MVT::nxv8f16)
7464 return Register();
7465 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7466 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7467 }
7468 return Register();
7469}
7470
7471Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7472 if (RetVT.SimpleTy != MVT::nxv4f32)
7473 return Register();
7474 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7475 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7476 }
7477 return Register();
7478}
7479
7480Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7481 if (RetVT.SimpleTy != MVT::nxv2f64)
7482 return Register();
7483 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7484 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7485 }
7486 return Register();
7487}
7488
7489Register fastEmit_AArch64ISD_FRSQRTS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7490 switch (VT.SimpleTy) {
7491 case MVT::f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(RetVT, Op0, Op1);
7492 case MVT::f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(RetVT, Op0, Op1);
7493 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(RetVT, Op0, Op1);
7494 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(RetVT, Op0, Op1);
7495 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(RetVT, Op0, Op1);
7496 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(RetVT, Op0, Op1);
7497 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(RetVT, Op0, Op1);
7498 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(RetVT, Op0, Op1);
7499 default: return Register();
7500 }
7501}
7502
7503// FastEmit functions for AArch64ISD::PMULL.
7504
7505Register fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7506 if (RetVT.SimpleTy != MVT::v8i16)
7507 return Register();
7508 if ((Subtarget->isNeonAvailable())) {
7509 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULLv8i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
7510 }
7511 return Register();
7512}
7513
7514Register fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7515 if (RetVT.SimpleTy != MVT::v16i8)
7516 return Register();
7517 if ((Subtarget->hasAES())) {
7518 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULLv1i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7519 }
7520 return Register();
7521}
7522
7523Register fastEmit_AArch64ISD_PMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7524 switch (VT.SimpleTy) {
7525 case MVT::v8i8: return fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
7526 case MVT::v1i64: return fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(RetVT, Op0, Op1);
7527 default: return Register();
7528 }
7529}
7530
7531// FastEmit functions for AArch64ISD::SMULL.
7532
7533Register fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7534 if (RetVT.SimpleTy != MVT::v8i16)
7535 return Register();
7536 if ((Subtarget->isNeonAvailable())) {
7537 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv8i8_v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7538 }
7539 return Register();
7540}
7541
7542Register fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7543 if (RetVT.SimpleTy != MVT::v4i32)
7544 return Register();
7545 if ((Subtarget->isNeonAvailable())) {
7546 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv4i16_v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7547 }
7548 return Register();
7549}
7550
7551Register fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7552 if (RetVT.SimpleTy != MVT::v2i64)
7553 return Register();
7554 if ((Subtarget->isNeonAvailable())) {
7555 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv2i32_v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7556 }
7557 return Register();
7558}
7559
7560Register fastEmit_AArch64ISD_SMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7561 switch (VT.SimpleTy) {
7562 case MVT::v8i8: return fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
7563 case MVT::v4i16: return fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(RetVT, Op0, Op1);
7564 case MVT::v2i32: return fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(RetVT, Op0, Op1);
7565 default: return Register();
7566 }
7567}
7568
7569// FastEmit functions for AArch64ISD::SQADD.
7570
7571Register fastEmit_AArch64ISD_SQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7572 if (RetVT.SimpleTy != MVT::f32)
7573 return Register();
7574 if ((Subtarget->isNeonAvailable())) {
7575 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7576 }
7577 return Register();
7578}
7579
7580Register fastEmit_AArch64ISD_SQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7581 if (RetVT.SimpleTy != MVT::f64)
7582 return Register();
7583 if ((Subtarget->isNeonAvailable())) {
7584 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7585 }
7586 return Register();
7587}
7588
7589Register fastEmit_AArch64ISD_SQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7590 switch (VT.SimpleTy) {
7591 case MVT::f32: return fastEmit_AArch64ISD_SQADD_MVT_f32_rr(RetVT, Op0, Op1);
7592 case MVT::f64: return fastEmit_AArch64ISD_SQADD_MVT_f64_rr(RetVT, Op0, Op1);
7593 default: return Register();
7594 }
7595}
7596
7597// FastEmit functions for AArch64ISD::SQDMULH.
7598
7599Register fastEmit_AArch64ISD_SQDMULH_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7600 if (RetVT.SimpleTy != MVT::f32)
7601 return Register();
7602 if ((Subtarget->isNeonAvailable())) {
7603 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7604 }
7605 return Register();
7606}
7607
7608Register fastEmit_AArch64ISD_SQDMULH_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7609 if (RetVT.SimpleTy != MVT::v4i16)
7610 return Register();
7611 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7612}
7613
7614Register fastEmit_AArch64ISD_SQDMULH_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7615 if (RetVT.SimpleTy != MVT::v8i16)
7616 return Register();
7617 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7618}
7619
7620Register fastEmit_AArch64ISD_SQDMULH_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7621 if (RetVT.SimpleTy != MVT::v2i32)
7622 return Register();
7623 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7624}
7625
7626Register fastEmit_AArch64ISD_SQDMULH_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7627 if (RetVT.SimpleTy != MVT::v4i32)
7628 return Register();
7629 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7630}
7631
7632Register fastEmit_AArch64ISD_SQDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7633 switch (VT.SimpleTy) {
7634 case MVT::f32: return fastEmit_AArch64ISD_SQDMULH_MVT_f32_rr(RetVT, Op0, Op1);
7635 case MVT::v4i16: return fastEmit_AArch64ISD_SQDMULH_MVT_v4i16_rr(RetVT, Op0, Op1);
7636 case MVT::v8i16: return fastEmit_AArch64ISD_SQDMULH_MVT_v8i16_rr(RetVT, Op0, Op1);
7637 case MVT::v2i32: return fastEmit_AArch64ISD_SQDMULH_MVT_v2i32_rr(RetVT, Op0, Op1);
7638 case MVT::v4i32: return fastEmit_AArch64ISD_SQDMULH_MVT_v4i32_rr(RetVT, Op0, Op1);
7639 default: return Register();
7640 }
7641}
7642
7643// FastEmit functions for AArch64ISD::SQDMULL.
7644
7645Register fastEmit_AArch64ISD_SQDMULL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7646 if (RetVT.SimpleTy != MVT::f64)
7647 return Register();
7648 if ((Subtarget->isNeonAvailable())) {
7649 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULLi32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7650 }
7651 return Register();
7652}
7653
7654Register fastEmit_AArch64ISD_SQDMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7655 switch (VT.SimpleTy) {
7656 case MVT::f32: return fastEmit_AArch64ISD_SQDMULL_MVT_f32_rr(RetVT, Op0, Op1);
7657 default: return Register();
7658 }
7659}
7660
7661// FastEmit functions for AArch64ISD::SQRDMULH.
7662
7663Register fastEmit_AArch64ISD_SQRDMULH_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7664 if (RetVT.SimpleTy != MVT::f32)
7665 return Register();
7666 if ((Subtarget->isNeonAvailable())) {
7667 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRDMULHv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7668 }
7669 return Register();
7670}
7671
7672Register fastEmit_AArch64ISD_SQRDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7673 switch (VT.SimpleTy) {
7674 case MVT::f32: return fastEmit_AArch64ISD_SQRDMULH_MVT_f32_rr(RetVT, Op0, Op1);
7675 default: return Register();
7676 }
7677}
7678
7679// FastEmit functions for AArch64ISD::SQRSHL.
7680
7681Register fastEmit_AArch64ISD_SQRSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7682 if (RetVT.SimpleTy != MVT::f32)
7683 return Register();
7684 if ((Subtarget->isNeonAvailable())) {
7685 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7686 }
7687 return Register();
7688}
7689
7690Register fastEmit_AArch64ISD_SQRSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7691 if (RetVT.SimpleTy != MVT::f64)
7692 return Register();
7693 if ((Subtarget->isNeonAvailable())) {
7694 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7695 }
7696 return Register();
7697}
7698
7699Register fastEmit_AArch64ISD_SQRSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7700 switch (VT.SimpleTy) {
7701 case MVT::f32: return fastEmit_AArch64ISD_SQRSHL_MVT_f32_rr(RetVT, Op0, Op1);
7702 case MVT::f64: return fastEmit_AArch64ISD_SQRSHL_MVT_f64_rr(RetVT, Op0, Op1);
7703 default: return Register();
7704 }
7705}
7706
7707// FastEmit functions for AArch64ISD::SQSHL.
7708
7709Register fastEmit_AArch64ISD_SQSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7710 if (RetVT.SimpleTy != MVT::f32)
7711 return Register();
7712 if ((Subtarget->isNeonAvailable())) {
7713 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7714 }
7715 return Register();
7716}
7717
7718Register fastEmit_AArch64ISD_SQSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7719 if (RetVT.SimpleTy != MVT::f64)
7720 return Register();
7721 if ((Subtarget->isNeonAvailable())) {
7722 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7723 }
7724 return Register();
7725}
7726
7727Register fastEmit_AArch64ISD_SQSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7728 switch (VT.SimpleTy) {
7729 case MVT::f32: return fastEmit_AArch64ISD_SQSHL_MVT_f32_rr(RetVT, Op0, Op1);
7730 case MVT::f64: return fastEmit_AArch64ISD_SQSHL_MVT_f64_rr(RetVT, Op0, Op1);
7731 default: return Register();
7732 }
7733}
7734
7735// FastEmit functions for AArch64ISD::SQSUB.
7736
7737Register fastEmit_AArch64ISD_SQSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7738 if (RetVT.SimpleTy != MVT::f32)
7739 return Register();
7740 if ((Subtarget->isNeonAvailable())) {
7741 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7742 }
7743 return Register();
7744}
7745
7746Register fastEmit_AArch64ISD_SQSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7747 if (RetVT.SimpleTy != MVT::f64)
7748 return Register();
7749 if ((Subtarget->isNeonAvailable())) {
7750 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7751 }
7752 return Register();
7753}
7754
7755Register fastEmit_AArch64ISD_SQSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7756 switch (VT.SimpleTy) {
7757 case MVT::f32: return fastEmit_AArch64ISD_SQSUB_MVT_f32_rr(RetVT, Op0, Op1);
7758 case MVT::f64: return fastEmit_AArch64ISD_SQSUB_MVT_f64_rr(RetVT, Op0, Op1);
7759 default: return Register();
7760 }
7761}
7762
7763// FastEmit functions for AArch64ISD::SUQADD.
7764
7765Register fastEmit_AArch64ISD_SUQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7766 if (RetVT.SimpleTy != MVT::f32)
7767 return Register();
7768 if ((Subtarget->isNeonAvailable())) {
7769 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7770 }
7771 return Register();
7772}
7773
7774Register fastEmit_AArch64ISD_SUQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7775 if (RetVT.SimpleTy != MVT::f64)
7776 return Register();
7777 if ((Subtarget->isNeonAvailable())) {
7778 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7779 }
7780 return Register();
7781}
7782
7783Register fastEmit_AArch64ISD_SUQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7784 switch (VT.SimpleTy) {
7785 case MVT::f32: return fastEmit_AArch64ISD_SUQADD_MVT_f32_rr(RetVT, Op0, Op1);
7786 case MVT::f64: return fastEmit_AArch64ISD_SUQADD_MVT_f64_rr(RetVT, Op0, Op1);
7787 default: return Register();
7788 }
7789}
7790
7791// FastEmit functions for AArch64ISD::TBL.
7792
7793Register fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7794 if (RetVT.SimpleTy != MVT::nxv16i8)
7795 return Register();
7796 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7797 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
7798 }
7799 return Register();
7800}
7801
7802Register fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7803 if (RetVT.SimpleTy != MVT::nxv8i16)
7804 return Register();
7805 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7806 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7807 }
7808 return Register();
7809}
7810
7811Register fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7812 if (RetVT.SimpleTy != MVT::nxv4i32)
7813 return Register();
7814 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7815 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7816 }
7817 return Register();
7818}
7819
7820Register fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7821 if (RetVT.SimpleTy != MVT::nxv2i64)
7822 return Register();
7823 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7824 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7825 }
7826 return Register();
7827}
7828
7829Register fastEmit_AArch64ISD_TBL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7830 switch (VT.SimpleTy) {
7831 case MVT::nxv16i8: return fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(RetVT, Op0, Op1);
7832 case MVT::nxv8i16: return fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(RetVT, Op0, Op1);
7833 case MVT::nxv4i32: return fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(RetVT, Op0, Op1);
7834 case MVT::nxv2i64: return fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(RetVT, Op0, Op1);
7835 default: return Register();
7836 }
7837}
7838
7839// FastEmit functions for AArch64ISD::TRN1.
7840
7841Register fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7842 if (RetVT.SimpleTy != MVT::v8i8)
7843 return Register();
7844 if ((Subtarget->isNeonAvailable())) {
7845 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
7846 }
7847 return Register();
7848}
7849
7850Register fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7851 if (RetVT.SimpleTy != MVT::v16i8)
7852 return Register();
7853 if ((Subtarget->isNeonAvailable())) {
7854 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
7855 }
7856 return Register();
7857}
7858
7859Register fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7860 if (RetVT.SimpleTy != MVT::v4i16)
7861 return Register();
7862 if ((Subtarget->isNeonAvailable())) {
7863 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7864 }
7865 return Register();
7866}
7867
7868Register fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7869 if (RetVT.SimpleTy != MVT::v8i16)
7870 return Register();
7871 if ((Subtarget->isNeonAvailable())) {
7872 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7873 }
7874 return Register();
7875}
7876
7877Register fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7878 if (RetVT.SimpleTy != MVT::v2i32)
7879 return Register();
7880 if ((Subtarget->isNeonAvailable())) {
7881 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7882 }
7883 return Register();
7884}
7885
7886Register fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7887 if (RetVT.SimpleTy != MVT::v4i32)
7888 return Register();
7889 if ((Subtarget->isNeonAvailable())) {
7890 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7891 }
7892 return Register();
7893}
7894
7895Register fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7896 if (RetVT.SimpleTy != MVT::v2i64)
7897 return Register();
7898 if ((Subtarget->isNeonAvailable())) {
7899 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7900 }
7901 return Register();
7902}
7903
7904Register fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7905 if (RetVT.SimpleTy != MVT::v4f16)
7906 return Register();
7907 if ((Subtarget->isNeonAvailable())) {
7908 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7909 }
7910 return Register();
7911}
7912
7913Register fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7914 if (RetVT.SimpleTy != MVT::v8f16)
7915 return Register();
7916 if ((Subtarget->isNeonAvailable())) {
7917 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7918 }
7919 return Register();
7920}
7921
7922Register fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
7923 if (RetVT.SimpleTy != MVT::v4bf16)
7924 return Register();
7925 if ((Subtarget->isNeonAvailable())) {
7926 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7927 }
7928 return Register();
7929}
7930
7931Register fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
7932 if (RetVT.SimpleTy != MVT::v8bf16)
7933 return Register();
7934 if ((Subtarget->isNeonAvailable())) {
7935 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7936 }
7937 return Register();
7938}
7939
7940Register fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7941 if (RetVT.SimpleTy != MVT::v2f32)
7942 return Register();
7943 if ((Subtarget->isNeonAvailable())) {
7944 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7945 }
7946 return Register();
7947}
7948
7949Register fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7950 if (RetVT.SimpleTy != MVT::v4f32)
7951 return Register();
7952 if ((Subtarget->isNeonAvailable())) {
7953 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7954 }
7955 return Register();
7956}
7957
7958Register fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7959 if (RetVT.SimpleTy != MVT::v2f64)
7960 return Register();
7961 if ((Subtarget->isNeonAvailable())) {
7962 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7963 }
7964 return Register();
7965}
7966
7967Register fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
7968 if (RetVT.SimpleTy != MVT::nxv2i1)
7969 return Register();
7970 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7971 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
7972 }
7973 return Register();
7974}
7975
7976Register fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
7977 if (RetVT.SimpleTy != MVT::nxv4i1)
7978 return Register();
7979 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7980 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
7981 }
7982 return Register();
7983}
7984
7985Register fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
7986 if (RetVT.SimpleTy != MVT::nxv8i1)
7987 return Register();
7988 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7989 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
7990 }
7991 return Register();
7992}
7993
7994Register fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7995 if (RetVT.SimpleTy != MVT::nxv16i1)
7996 return Register();
7997 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7998 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
7999 }
8000 return Register();
8001}
8002
8003Register fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8004 if (RetVT.SimpleTy != MVT::nxv16i8)
8005 return Register();
8006 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8007 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8008 }
8009 return Register();
8010}
8011
8012Register fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8013 if (RetVT.SimpleTy != MVT::nxv8i16)
8014 return Register();
8015 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8016 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8017 }
8018 return Register();
8019}
8020
8021Register fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8022 if (RetVT.SimpleTy != MVT::nxv4i32)
8023 return Register();
8024 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8025 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8026 }
8027 return Register();
8028}
8029
8030Register fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8031 if (RetVT.SimpleTy != MVT::nxv2i64)
8032 return Register();
8033 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8034 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8035 }
8036 return Register();
8037}
8038
8039Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8040 if (RetVT.SimpleTy != MVT::nxv2f16)
8041 return Register();
8042 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8043 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8044 }
8045 return Register();
8046}
8047
8048Register fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8049 if (RetVT.SimpleTy != MVT::nxv4f16)
8050 return Register();
8051 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8052 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8053 }
8054 return Register();
8055}
8056
8057Register fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8058 if (RetVT.SimpleTy != MVT::nxv8f16)
8059 return Register();
8060 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8061 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8062 }
8063 return Register();
8064}
8065
8066Register fastEmit_AArch64ISD_TRN1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8067 if (RetVT.SimpleTy != MVT::nxv2bf16)
8068 return Register();
8069 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8070 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8071 }
8072 return Register();
8073}
8074
8075Register fastEmit_AArch64ISD_TRN1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8076 if (RetVT.SimpleTy != MVT::nxv4bf16)
8077 return Register();
8078 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8079 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8080 }
8081 return Register();
8082}
8083
8084Register fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8085 if (RetVT.SimpleTy != MVT::nxv8bf16)
8086 return Register();
8087 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8088 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8089 }
8090 return Register();
8091}
8092
8093Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8094 if (RetVT.SimpleTy != MVT::nxv2f32)
8095 return Register();
8096 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8097 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8098 }
8099 return Register();
8100}
8101
8102Register fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8103 if (RetVT.SimpleTy != MVT::nxv4f32)
8104 return Register();
8105 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8106 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8107 }
8108 return Register();
8109}
8110
8111Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8112 if (RetVT.SimpleTy != MVT::nxv2f64)
8113 return Register();
8114 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8115 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8116 }
8117 return Register();
8118}
8119
8120Register fastEmit_AArch64ISD_TRN1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8121 switch (VT.SimpleTy) {
8122 case MVT::v8i8: return fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(RetVT, Op0, Op1);
8123 case MVT::v16i8: return fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(RetVT, Op0, Op1);
8124 case MVT::v4i16: return fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(RetVT, Op0, Op1);
8125 case MVT::v8i16: return fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(RetVT, Op0, Op1);
8126 case MVT::v2i32: return fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(RetVT, Op0, Op1);
8127 case MVT::v4i32: return fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(RetVT, Op0, Op1);
8128 case MVT::v2i64: return fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(RetVT, Op0, Op1);
8129 case MVT::v4f16: return fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(RetVT, Op0, Op1);
8130 case MVT::v8f16: return fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(RetVT, Op0, Op1);
8131 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(RetVT, Op0, Op1);
8132 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(RetVT, Op0, Op1);
8133 case MVT::v2f32: return fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(RetVT, Op0, Op1);
8134 case MVT::v4f32: return fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(RetVT, Op0, Op1);
8135 case MVT::v2f64: return fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(RetVT, Op0, Op1);
8136 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8137 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8138 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8139 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8140 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8141 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8142 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8143 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8144 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8145 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8146 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8147 case MVT::nxv2bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8148 case MVT::nxv4bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8149 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8150 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8151 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8152 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8153 default: return Register();
8154 }
8155}
8156
8157// FastEmit functions for AArch64ISD::TRN2.
8158
8159Register fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8160 if (RetVT.SimpleTy != MVT::v8i8)
8161 return Register();
8162 if ((Subtarget->isNeonAvailable())) {
8163 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8164 }
8165 return Register();
8166}
8167
8168Register fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8169 if (RetVT.SimpleTy != MVT::v16i8)
8170 return Register();
8171 if ((Subtarget->isNeonAvailable())) {
8172 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8173 }
8174 return Register();
8175}
8176
8177Register fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8178 if (RetVT.SimpleTy != MVT::v4i16)
8179 return Register();
8180 if ((Subtarget->isNeonAvailable())) {
8181 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8182 }
8183 return Register();
8184}
8185
8186Register fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8187 if (RetVT.SimpleTy != MVT::v8i16)
8188 return Register();
8189 if ((Subtarget->isNeonAvailable())) {
8190 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8191 }
8192 return Register();
8193}
8194
8195Register fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8196 if (RetVT.SimpleTy != MVT::v2i32)
8197 return Register();
8198 if ((Subtarget->isNeonAvailable())) {
8199 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8200 }
8201 return Register();
8202}
8203
8204Register fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8205 if (RetVT.SimpleTy != MVT::v4i32)
8206 return Register();
8207 if ((Subtarget->isNeonAvailable())) {
8208 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8209 }
8210 return Register();
8211}
8212
8213Register fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8214 if (RetVT.SimpleTy != MVT::v2i64)
8215 return Register();
8216 if ((Subtarget->isNeonAvailable())) {
8217 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8218 }
8219 return Register();
8220}
8221
8222Register fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8223 if (RetVT.SimpleTy != MVT::v4f16)
8224 return Register();
8225 if ((Subtarget->isNeonAvailable())) {
8226 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8227 }
8228 return Register();
8229}
8230
8231Register fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8232 if (RetVT.SimpleTy != MVT::v8f16)
8233 return Register();
8234 if ((Subtarget->isNeonAvailable())) {
8235 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8236 }
8237 return Register();
8238}
8239
8240Register fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8241 if (RetVT.SimpleTy != MVT::v4bf16)
8242 return Register();
8243 if ((Subtarget->isNeonAvailable())) {
8244 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8245 }
8246 return Register();
8247}
8248
8249Register fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8250 if (RetVT.SimpleTy != MVT::v8bf16)
8251 return Register();
8252 if ((Subtarget->isNeonAvailable())) {
8253 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8254 }
8255 return Register();
8256}
8257
8258Register fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8259 if (RetVT.SimpleTy != MVT::v2f32)
8260 return Register();
8261 if ((Subtarget->isNeonAvailable())) {
8262 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8263 }
8264 return Register();
8265}
8266
8267Register fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8268 if (RetVT.SimpleTy != MVT::v4f32)
8269 return Register();
8270 if ((Subtarget->isNeonAvailable())) {
8271 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8272 }
8273 return Register();
8274}
8275
8276Register fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8277 if (RetVT.SimpleTy != MVT::v2f64)
8278 return Register();
8279 if ((Subtarget->isNeonAvailable())) {
8280 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8281 }
8282 return Register();
8283}
8284
8285Register fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8286 if (RetVT.SimpleTy != MVT::nxv2i1)
8287 return Register();
8288 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8289 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8290 }
8291 return Register();
8292}
8293
8294Register fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8295 if (RetVT.SimpleTy != MVT::nxv4i1)
8296 return Register();
8297 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8298 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8299 }
8300 return Register();
8301}
8302
8303Register fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8304 if (RetVT.SimpleTy != MVT::nxv8i1)
8305 return Register();
8306 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8307 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8308 }
8309 return Register();
8310}
8311
8312Register fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8313 if (RetVT.SimpleTy != MVT::nxv16i1)
8314 return Register();
8315 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8316 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8317 }
8318 return Register();
8319}
8320
8321Register fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8322 if (RetVT.SimpleTy != MVT::nxv16i8)
8323 return Register();
8324 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8325 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8326 }
8327 return Register();
8328}
8329
8330Register fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8331 if (RetVT.SimpleTy != MVT::nxv8i16)
8332 return Register();
8333 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8334 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8335 }
8336 return Register();
8337}
8338
8339Register fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8340 if (RetVT.SimpleTy != MVT::nxv4i32)
8341 return Register();
8342 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8343 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8344 }
8345 return Register();
8346}
8347
8348Register fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8349 if (RetVT.SimpleTy != MVT::nxv2i64)
8350 return Register();
8351 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8352 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8353 }
8354 return Register();
8355}
8356
8357Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8358 if (RetVT.SimpleTy != MVT::nxv2f16)
8359 return Register();
8360 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8361 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8362 }
8363 return Register();
8364}
8365
8366Register fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8367 if (RetVT.SimpleTy != MVT::nxv4f16)
8368 return Register();
8369 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8370 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8371 }
8372 return Register();
8373}
8374
8375Register fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8376 if (RetVT.SimpleTy != MVT::nxv8f16)
8377 return Register();
8378 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8379 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8380 }
8381 return Register();
8382}
8383
8384Register fastEmit_AArch64ISD_TRN2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8385 if (RetVT.SimpleTy != MVT::nxv2bf16)
8386 return Register();
8387 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8388 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8389 }
8390 return Register();
8391}
8392
8393Register fastEmit_AArch64ISD_TRN2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8394 if (RetVT.SimpleTy != MVT::nxv4bf16)
8395 return Register();
8396 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8397 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8398 }
8399 return Register();
8400}
8401
8402Register fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8403 if (RetVT.SimpleTy != MVT::nxv8bf16)
8404 return Register();
8405 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8406 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8407 }
8408 return Register();
8409}
8410
8411Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8412 if (RetVT.SimpleTy != MVT::nxv2f32)
8413 return Register();
8414 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8415 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8416 }
8417 return Register();
8418}
8419
8420Register fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8421 if (RetVT.SimpleTy != MVT::nxv4f32)
8422 return Register();
8423 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8424 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8425 }
8426 return Register();
8427}
8428
8429Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8430 if (RetVT.SimpleTy != MVT::nxv2f64)
8431 return Register();
8432 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8433 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8434 }
8435 return Register();
8436}
8437
8438Register fastEmit_AArch64ISD_TRN2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8439 switch (VT.SimpleTy) {
8440 case MVT::v8i8: return fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(RetVT, Op0, Op1);
8441 case MVT::v16i8: return fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(RetVT, Op0, Op1);
8442 case MVT::v4i16: return fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(RetVT, Op0, Op1);
8443 case MVT::v8i16: return fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(RetVT, Op0, Op1);
8444 case MVT::v2i32: return fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(RetVT, Op0, Op1);
8445 case MVT::v4i32: return fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(RetVT, Op0, Op1);
8446 case MVT::v2i64: return fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(RetVT, Op0, Op1);
8447 case MVT::v4f16: return fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(RetVT, Op0, Op1);
8448 case MVT::v8f16: return fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(RetVT, Op0, Op1);
8449 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(RetVT, Op0, Op1);
8450 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(RetVT, Op0, Op1);
8451 case MVT::v2f32: return fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(RetVT, Op0, Op1);
8452 case MVT::v4f32: return fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(RetVT, Op0, Op1);
8453 case MVT::v2f64: return fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(RetVT, Op0, Op1);
8454 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8455 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8456 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8457 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8458 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8459 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8460 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8461 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8462 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8463 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8464 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8465 case MVT::nxv2bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8466 case MVT::nxv4bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8467 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8468 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8469 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8470 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8471 default: return Register();
8472 }
8473}
8474
8475// FastEmit functions for AArch64ISD::UMULL.
8476
8477Register fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8478 if (RetVT.SimpleTy != MVT::v8i16)
8479 return Register();
8480 if ((Subtarget->isNeonAvailable())) {
8481 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv8i8_v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8482 }
8483 return Register();
8484}
8485
8486Register fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8487 if (RetVT.SimpleTy != MVT::v4i32)
8488 return Register();
8489 if ((Subtarget->isNeonAvailable())) {
8490 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv4i16_v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8491 }
8492 return Register();
8493}
8494
8495Register fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8496 if (RetVT.SimpleTy != MVT::v2i64)
8497 return Register();
8498 if ((Subtarget->isNeonAvailable())) {
8499 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv2i32_v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8500 }
8501 return Register();
8502}
8503
8504Register fastEmit_AArch64ISD_UMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8505 switch (VT.SimpleTy) {
8506 case MVT::v8i8: return fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
8507 case MVT::v4i16: return fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(RetVT, Op0, Op1);
8508 case MVT::v2i32: return fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(RetVT, Op0, Op1);
8509 default: return Register();
8510 }
8511}
8512
8513// FastEmit functions for AArch64ISD::UQADD.
8514
8515Register fastEmit_AArch64ISD_UQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8516 if (RetVT.SimpleTy != MVT::f32)
8517 return Register();
8518 if ((Subtarget->isNeonAvailable())) {
8519 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8520 }
8521 return Register();
8522}
8523
8524Register fastEmit_AArch64ISD_UQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8525 if (RetVT.SimpleTy != MVT::f64)
8526 return Register();
8527 if ((Subtarget->isNeonAvailable())) {
8528 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8529 }
8530 return Register();
8531}
8532
8533Register fastEmit_AArch64ISD_UQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8534 switch (VT.SimpleTy) {
8535 case MVT::f32: return fastEmit_AArch64ISD_UQADD_MVT_f32_rr(RetVT, Op0, Op1);
8536 case MVT::f64: return fastEmit_AArch64ISD_UQADD_MVT_f64_rr(RetVT, Op0, Op1);
8537 default: return Register();
8538 }
8539}
8540
8541// FastEmit functions for AArch64ISD::UQRSHL.
8542
8543Register fastEmit_AArch64ISD_UQRSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8544 if (RetVT.SimpleTy != MVT::f32)
8545 return Register();
8546 if ((Subtarget->isNeonAvailable())) {
8547 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQRSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8548 }
8549 return Register();
8550}
8551
8552Register fastEmit_AArch64ISD_UQRSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8553 if (RetVT.SimpleTy != MVT::f64)
8554 return Register();
8555 if ((Subtarget->isNeonAvailable())) {
8556 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQRSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8557 }
8558 return Register();
8559}
8560
8561Register fastEmit_AArch64ISD_UQRSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8562 switch (VT.SimpleTy) {
8563 case MVT::f32: return fastEmit_AArch64ISD_UQRSHL_MVT_f32_rr(RetVT, Op0, Op1);
8564 case MVT::f64: return fastEmit_AArch64ISD_UQRSHL_MVT_f64_rr(RetVT, Op0, Op1);
8565 default: return Register();
8566 }
8567}
8568
8569// FastEmit functions for AArch64ISD::UQSHL.
8570
8571Register fastEmit_AArch64ISD_UQSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8572 if (RetVT.SimpleTy != MVT::f32)
8573 return Register();
8574 if ((Subtarget->isNeonAvailable())) {
8575 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8576 }
8577 return Register();
8578}
8579
8580Register fastEmit_AArch64ISD_UQSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8581 if (RetVT.SimpleTy != MVT::f64)
8582 return Register();
8583 if ((Subtarget->isNeonAvailable())) {
8584 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8585 }
8586 return Register();
8587}
8588
8589Register fastEmit_AArch64ISD_UQSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8590 switch (VT.SimpleTy) {
8591 case MVT::f32: return fastEmit_AArch64ISD_UQSHL_MVT_f32_rr(RetVT, Op0, Op1);
8592 case MVT::f64: return fastEmit_AArch64ISD_UQSHL_MVT_f64_rr(RetVT, Op0, Op1);
8593 default: return Register();
8594 }
8595}
8596
8597// FastEmit functions for AArch64ISD::UQSUB.
8598
8599Register fastEmit_AArch64ISD_UQSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8600 if (RetVT.SimpleTy != MVT::f32)
8601 return Register();
8602 if ((Subtarget->isNeonAvailable())) {
8603 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8604 }
8605 return Register();
8606}
8607
8608Register fastEmit_AArch64ISD_UQSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8609 if (RetVT.SimpleTy != MVT::f64)
8610 return Register();
8611 if ((Subtarget->isNeonAvailable())) {
8612 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8613 }
8614 return Register();
8615}
8616
8617Register fastEmit_AArch64ISD_UQSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8618 switch (VT.SimpleTy) {
8619 case MVT::f32: return fastEmit_AArch64ISD_UQSUB_MVT_f32_rr(RetVT, Op0, Op1);
8620 case MVT::f64: return fastEmit_AArch64ISD_UQSUB_MVT_f64_rr(RetVT, Op0, Op1);
8621 default: return Register();
8622 }
8623}
8624
8625// FastEmit functions for AArch64ISD::USQADD.
8626
8627Register fastEmit_AArch64ISD_USQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8628 if (RetVT.SimpleTy != MVT::f32)
8629 return Register();
8630 if ((Subtarget->isNeonAvailable())) {
8631 return fastEmitInst_rr(MachineInstOpcode: AArch64::USQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8632 }
8633 return Register();
8634}
8635
8636Register fastEmit_AArch64ISD_USQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8637 if (RetVT.SimpleTy != MVT::f64)
8638 return Register();
8639 if ((Subtarget->isNeonAvailable())) {
8640 return fastEmitInst_rr(MachineInstOpcode: AArch64::USQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8641 }
8642 return Register();
8643}
8644
8645Register fastEmit_AArch64ISD_USQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8646 switch (VT.SimpleTy) {
8647 case MVT::f32: return fastEmit_AArch64ISD_USQADD_MVT_f32_rr(RetVT, Op0, Op1);
8648 case MVT::f64: return fastEmit_AArch64ISD_USQADD_MVT_f64_rr(RetVT, Op0, Op1);
8649 default: return Register();
8650 }
8651}
8652
8653// FastEmit functions for AArch64ISD::UZP1.
8654
8655Register fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8656 if (RetVT.SimpleTy != MVT::v8i8)
8657 return Register();
8658 if ((Subtarget->isNeonAvailable())) {
8659 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8660 }
8661 return Register();
8662}
8663
8664Register fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8665 if (RetVT.SimpleTy != MVT::v16i8)
8666 return Register();
8667 if ((Subtarget->isNeonAvailable())) {
8668 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8669 }
8670 return Register();
8671}
8672
8673Register fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8674 if (RetVT.SimpleTy != MVT::v4i16)
8675 return Register();
8676 if ((Subtarget->isNeonAvailable())) {
8677 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8678 }
8679 return Register();
8680}
8681
8682Register fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8683 if (RetVT.SimpleTy != MVT::v8i16)
8684 return Register();
8685 if ((Subtarget->isNeonAvailable())) {
8686 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8687 }
8688 return Register();
8689}
8690
8691Register fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8692 if (RetVT.SimpleTy != MVT::v2i32)
8693 return Register();
8694 if ((Subtarget->isNeonAvailable())) {
8695 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8696 }
8697 return Register();
8698}
8699
8700Register fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8701 if (RetVT.SimpleTy != MVT::v4i32)
8702 return Register();
8703 if ((Subtarget->isNeonAvailable())) {
8704 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8705 }
8706 return Register();
8707}
8708
8709Register fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8710 if (RetVT.SimpleTy != MVT::v2i64)
8711 return Register();
8712 if ((Subtarget->isNeonAvailable())) {
8713 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8714 }
8715 return Register();
8716}
8717
8718Register fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8719 if (RetVT.SimpleTy != MVT::v4f16)
8720 return Register();
8721 if ((Subtarget->isNeonAvailable())) {
8722 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8723 }
8724 return Register();
8725}
8726
8727Register fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8728 if (RetVT.SimpleTy != MVT::v8f16)
8729 return Register();
8730 if ((Subtarget->isNeonAvailable())) {
8731 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8732 }
8733 return Register();
8734}
8735
8736Register fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8737 if (RetVT.SimpleTy != MVT::v4bf16)
8738 return Register();
8739 if ((Subtarget->isNeonAvailable())) {
8740 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8741 }
8742 return Register();
8743}
8744
8745Register fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8746 if (RetVT.SimpleTy != MVT::v8bf16)
8747 return Register();
8748 if ((Subtarget->isNeonAvailable())) {
8749 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8750 }
8751 return Register();
8752}
8753
8754Register fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8755 if (RetVT.SimpleTy != MVT::v2f32)
8756 return Register();
8757 if ((Subtarget->isNeonAvailable())) {
8758 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8759 }
8760 return Register();
8761}
8762
8763Register fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8764 if (RetVT.SimpleTy != MVT::v4f32)
8765 return Register();
8766 if ((Subtarget->isNeonAvailable())) {
8767 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8768 }
8769 return Register();
8770}
8771
8772Register fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8773 if (RetVT.SimpleTy != MVT::v2f64)
8774 return Register();
8775 if ((Subtarget->isNeonAvailable())) {
8776 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8777 }
8778 return Register();
8779}
8780
8781Register fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8782 if (RetVT.SimpleTy != MVT::nxv2i1)
8783 return Register();
8784 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8785 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8786 }
8787 return Register();
8788}
8789
8790Register fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8791 if (RetVT.SimpleTy != MVT::nxv4i1)
8792 return Register();
8793 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8794 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8795 }
8796 return Register();
8797}
8798
8799Register fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8800 if (RetVT.SimpleTy != MVT::nxv8i1)
8801 return Register();
8802 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8803 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8804 }
8805 return Register();
8806}
8807
8808Register fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8809 if (RetVT.SimpleTy != MVT::nxv16i1)
8810 return Register();
8811 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8812 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8813 }
8814 return Register();
8815}
8816
8817Register fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8818 if (RetVT.SimpleTy != MVT::nxv16i8)
8819 return Register();
8820 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8821 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8822 }
8823 return Register();
8824}
8825
8826Register fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8827 if (RetVT.SimpleTy != MVT::nxv8i16)
8828 return Register();
8829 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8830 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8831 }
8832 return Register();
8833}
8834
8835Register fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8836 if (RetVT.SimpleTy != MVT::nxv4i32)
8837 return Register();
8838 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8839 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8840 }
8841 return Register();
8842}
8843
8844Register fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8845 if (RetVT.SimpleTy != MVT::nxv2i64)
8846 return Register();
8847 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8848 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8849 }
8850 return Register();
8851}
8852
8853Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8854 if (RetVT.SimpleTy != MVT::nxv2f16)
8855 return Register();
8856 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8857 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8858 }
8859 return Register();
8860}
8861
8862Register fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8863 if (RetVT.SimpleTy != MVT::nxv4f16)
8864 return Register();
8865 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8866 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8867 }
8868 return Register();
8869}
8870
8871Register fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8872 if (RetVT.SimpleTy != MVT::nxv8f16)
8873 return Register();
8874 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8875 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8876 }
8877 return Register();
8878}
8879
8880Register fastEmit_AArch64ISD_UZP1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8881 if (RetVT.SimpleTy != MVT::nxv2bf16)
8882 return Register();
8883 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8884 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8885 }
8886 return Register();
8887}
8888
8889Register fastEmit_AArch64ISD_UZP1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8890 if (RetVT.SimpleTy != MVT::nxv4bf16)
8891 return Register();
8892 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8893 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8894 }
8895 return Register();
8896}
8897
8898Register fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8899 if (RetVT.SimpleTy != MVT::nxv8bf16)
8900 return Register();
8901 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8902 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8903 }
8904 return Register();
8905}
8906
8907Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8908 if (RetVT.SimpleTy != MVT::nxv2f32)
8909 return Register();
8910 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8911 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8912 }
8913 return Register();
8914}
8915
8916Register fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8917 if (RetVT.SimpleTy != MVT::nxv4f32)
8918 return Register();
8919 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8920 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8921 }
8922 return Register();
8923}
8924
8925Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8926 if (RetVT.SimpleTy != MVT::nxv2f64)
8927 return Register();
8928 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8929 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8930 }
8931 return Register();
8932}
8933
8934Register fastEmit_AArch64ISD_UZP1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8935 switch (VT.SimpleTy) {
8936 case MVT::v8i8: return fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(RetVT, Op0, Op1);
8937 case MVT::v16i8: return fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(RetVT, Op0, Op1);
8938 case MVT::v4i16: return fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(RetVT, Op0, Op1);
8939 case MVT::v8i16: return fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(RetVT, Op0, Op1);
8940 case MVT::v2i32: return fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(RetVT, Op0, Op1);
8941 case MVT::v4i32: return fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(RetVT, Op0, Op1);
8942 case MVT::v2i64: return fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(RetVT, Op0, Op1);
8943 case MVT::v4f16: return fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(RetVT, Op0, Op1);
8944 case MVT::v8f16: return fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(RetVT, Op0, Op1);
8945 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(RetVT, Op0, Op1);
8946 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(RetVT, Op0, Op1);
8947 case MVT::v2f32: return fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(RetVT, Op0, Op1);
8948 case MVT::v4f32: return fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(RetVT, Op0, Op1);
8949 case MVT::v2f64: return fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(RetVT, Op0, Op1);
8950 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8951 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8952 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8953 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8954 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8955 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8956 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8957 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8958 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8959 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8960 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8961 case MVT::nxv2bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8962 case MVT::nxv4bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8963 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8964 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8965 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8966 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8967 default: return Register();
8968 }
8969}
8970
8971// FastEmit functions for AArch64ISD::UZP2.
8972
8973Register fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8974 if (RetVT.SimpleTy != MVT::v8i8)
8975 return Register();
8976 if ((Subtarget->isNeonAvailable())) {
8977 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8978 }
8979 return Register();
8980}
8981
8982Register fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8983 if (RetVT.SimpleTy != MVT::v16i8)
8984 return Register();
8985 if ((Subtarget->isNeonAvailable())) {
8986 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8987 }
8988 return Register();
8989}
8990
8991Register fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8992 if (RetVT.SimpleTy != MVT::v4i16)
8993 return Register();
8994 if ((Subtarget->isNeonAvailable())) {
8995 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8996 }
8997 return Register();
8998}
8999
9000Register fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9001 if (RetVT.SimpleTy != MVT::v8i16)
9002 return Register();
9003 if ((Subtarget->isNeonAvailable())) {
9004 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9005 }
9006 return Register();
9007}
9008
9009Register fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9010 if (RetVT.SimpleTy != MVT::v2i32)
9011 return Register();
9012 if ((Subtarget->isNeonAvailable())) {
9013 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9014 }
9015 return Register();
9016}
9017
9018Register fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9019 if (RetVT.SimpleTy != MVT::v4i32)
9020 return Register();
9021 if ((Subtarget->isNeonAvailable())) {
9022 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9023 }
9024 return Register();
9025}
9026
9027Register fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9028 if (RetVT.SimpleTy != MVT::v2i64)
9029 return Register();
9030 if ((Subtarget->isNeonAvailable())) {
9031 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9032 }
9033 return Register();
9034}
9035
9036Register fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9037 if (RetVT.SimpleTy != MVT::v4f16)
9038 return Register();
9039 if ((Subtarget->isNeonAvailable())) {
9040 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9041 }
9042 return Register();
9043}
9044
9045Register fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9046 if (RetVT.SimpleTy != MVT::v8f16)
9047 return Register();
9048 if ((Subtarget->isNeonAvailable())) {
9049 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9050 }
9051 return Register();
9052}
9053
9054Register fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9055 if (RetVT.SimpleTy != MVT::v4bf16)
9056 return Register();
9057 if ((Subtarget->isNeonAvailable())) {
9058 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9059 }
9060 return Register();
9061}
9062
9063Register fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9064 if (RetVT.SimpleTy != MVT::v8bf16)
9065 return Register();
9066 if ((Subtarget->isNeonAvailable())) {
9067 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9068 }
9069 return Register();
9070}
9071
9072Register fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9073 if (RetVT.SimpleTy != MVT::v2f32)
9074 return Register();
9075 if ((Subtarget->isNeonAvailable())) {
9076 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9077 }
9078 return Register();
9079}
9080
9081Register fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9082 if (RetVT.SimpleTy != MVT::v4f32)
9083 return Register();
9084 if ((Subtarget->isNeonAvailable())) {
9085 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9086 }
9087 return Register();
9088}
9089
9090Register fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9091 if (RetVT.SimpleTy != MVT::v2f64)
9092 return Register();
9093 if ((Subtarget->isNeonAvailable())) {
9094 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9095 }
9096 return Register();
9097}
9098
9099Register fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9100 if (RetVT.SimpleTy != MVT::nxv2i1)
9101 return Register();
9102 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9103 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9104 }
9105 return Register();
9106}
9107
9108Register fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9109 if (RetVT.SimpleTy != MVT::nxv4i1)
9110 return Register();
9111 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9112 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9113 }
9114 return Register();
9115}
9116
9117Register fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9118 if (RetVT.SimpleTy != MVT::nxv8i1)
9119 return Register();
9120 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9121 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9122 }
9123 return Register();
9124}
9125
9126Register fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9127 if (RetVT.SimpleTy != MVT::nxv16i1)
9128 return Register();
9129 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9130 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9131 }
9132 return Register();
9133}
9134
9135Register fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9136 if (RetVT.SimpleTy != MVT::nxv16i8)
9137 return Register();
9138 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9139 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9140 }
9141 return Register();
9142}
9143
9144Register fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9145 if (RetVT.SimpleTy != MVT::nxv8i16)
9146 return Register();
9147 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9148 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9149 }
9150 return Register();
9151}
9152
9153Register fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9154 if (RetVT.SimpleTy != MVT::nxv4i32)
9155 return Register();
9156 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9157 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9158 }
9159 return Register();
9160}
9161
9162Register fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9163 if (RetVT.SimpleTy != MVT::nxv2i64)
9164 return Register();
9165 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9166 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9167 }
9168 return Register();
9169}
9170
9171Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9172 if (RetVT.SimpleTy != MVT::nxv2f16)
9173 return Register();
9174 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9175 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9176 }
9177 return Register();
9178}
9179
9180Register fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9181 if (RetVT.SimpleTy != MVT::nxv4f16)
9182 return Register();
9183 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9184 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9185 }
9186 return Register();
9187}
9188
9189Register fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9190 if (RetVT.SimpleTy != MVT::nxv8f16)
9191 return Register();
9192 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9193 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9194 }
9195 return Register();
9196}
9197
9198Register fastEmit_AArch64ISD_UZP2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9199 if (RetVT.SimpleTy != MVT::nxv2bf16)
9200 return Register();
9201 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9202 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9203 }
9204 return Register();
9205}
9206
9207Register fastEmit_AArch64ISD_UZP2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9208 if (RetVT.SimpleTy != MVT::nxv4bf16)
9209 return Register();
9210 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9211 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9212 }
9213 return Register();
9214}
9215
9216Register fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9217 if (RetVT.SimpleTy != MVT::nxv8bf16)
9218 return Register();
9219 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9220 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9221 }
9222 return Register();
9223}
9224
9225Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9226 if (RetVT.SimpleTy != MVT::nxv2f32)
9227 return Register();
9228 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9229 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9230 }
9231 return Register();
9232}
9233
9234Register fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9235 if (RetVT.SimpleTy != MVT::nxv4f32)
9236 return Register();
9237 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9238 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9239 }
9240 return Register();
9241}
9242
9243Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9244 if (RetVT.SimpleTy != MVT::nxv2f64)
9245 return Register();
9246 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9247 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9248 }
9249 return Register();
9250}
9251
9252Register fastEmit_AArch64ISD_UZP2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9253 switch (VT.SimpleTy) {
9254 case MVT::v8i8: return fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(RetVT, Op0, Op1);
9255 case MVT::v16i8: return fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(RetVT, Op0, Op1);
9256 case MVT::v4i16: return fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(RetVT, Op0, Op1);
9257 case MVT::v8i16: return fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(RetVT, Op0, Op1);
9258 case MVT::v2i32: return fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(RetVT, Op0, Op1);
9259 case MVT::v4i32: return fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(RetVT, Op0, Op1);
9260 case MVT::v2i64: return fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(RetVT, Op0, Op1);
9261 case MVT::v4f16: return fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(RetVT, Op0, Op1);
9262 case MVT::v8f16: return fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(RetVT, Op0, Op1);
9263 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(RetVT, Op0, Op1);
9264 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(RetVT, Op0, Op1);
9265 case MVT::v2f32: return fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(RetVT, Op0, Op1);
9266 case MVT::v4f32: return fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(RetVT, Op0, Op1);
9267 case MVT::v2f64: return fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(RetVT, Op0, Op1);
9268 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9269 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9270 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9271 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9272 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9273 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9274 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9275 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9276 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9277 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9278 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9279 case MVT::nxv2bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9280 case MVT::nxv4bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9281 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9282 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9283 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9284 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9285 default: return Register();
9286 }
9287}
9288
9289// FastEmit functions for AArch64ISD::ZIP1.
9290
9291Register fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9292 if (RetVT.SimpleTy != MVT::v8i8)
9293 return Register();
9294 if ((Subtarget->isNeonAvailable())) {
9295 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9296 }
9297 return Register();
9298}
9299
9300Register fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9301 if (RetVT.SimpleTy != MVT::v16i8)
9302 return Register();
9303 if ((Subtarget->isNeonAvailable())) {
9304 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9305 }
9306 return Register();
9307}
9308
9309Register fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9310 if (RetVT.SimpleTy != MVT::v4i16)
9311 return Register();
9312 if ((Subtarget->isNeonAvailable())) {
9313 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9314 }
9315 return Register();
9316}
9317
9318Register fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9319 if (RetVT.SimpleTy != MVT::v8i16)
9320 return Register();
9321 if ((Subtarget->isNeonAvailable())) {
9322 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9323 }
9324 return Register();
9325}
9326
9327Register fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9328 if (RetVT.SimpleTy != MVT::v2i32)
9329 return Register();
9330 if ((Subtarget->isNeonAvailable())) {
9331 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9332 }
9333 return Register();
9334}
9335
9336Register fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9337 if (RetVT.SimpleTy != MVT::v4i32)
9338 return Register();
9339 if ((Subtarget->isNeonAvailable())) {
9340 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9341 }
9342 return Register();
9343}
9344
9345Register fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9346 if (RetVT.SimpleTy != MVT::v2i64)
9347 return Register();
9348 if ((Subtarget->isNeonAvailable())) {
9349 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9350 }
9351 return Register();
9352}
9353
9354Register fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9355 if (RetVT.SimpleTy != MVT::v4f16)
9356 return Register();
9357 if ((Subtarget->isNeonAvailable())) {
9358 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9359 }
9360 return Register();
9361}
9362
9363Register fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9364 if (RetVT.SimpleTy != MVT::v8f16)
9365 return Register();
9366 if ((Subtarget->isNeonAvailable())) {
9367 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9368 }
9369 return Register();
9370}
9371
9372Register fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9373 if (RetVT.SimpleTy != MVT::v4bf16)
9374 return Register();
9375 if ((Subtarget->isNeonAvailable())) {
9376 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9377 }
9378 return Register();
9379}
9380
9381Register fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9382 if (RetVT.SimpleTy != MVT::v8bf16)
9383 return Register();
9384 if ((Subtarget->isNeonAvailable())) {
9385 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9386 }
9387 return Register();
9388}
9389
9390Register fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9391 if (RetVT.SimpleTy != MVT::v2f32)
9392 return Register();
9393 if ((Subtarget->isNeonAvailable())) {
9394 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9395 }
9396 return Register();
9397}
9398
9399Register fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9400 if (RetVT.SimpleTy != MVT::v4f32)
9401 return Register();
9402 if ((Subtarget->isNeonAvailable())) {
9403 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9404 }
9405 return Register();
9406}
9407
9408Register fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9409 if (RetVT.SimpleTy != MVT::v2f64)
9410 return Register();
9411 if ((Subtarget->isNeonAvailable())) {
9412 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9413 }
9414 return Register();
9415}
9416
9417Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9418 if (RetVT.SimpleTy != MVT::nxv2i1)
9419 return Register();
9420 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9421 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9422 }
9423 return Register();
9424}
9425
9426Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9427 if (RetVT.SimpleTy != MVT::nxv4i1)
9428 return Register();
9429 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9430 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9431 }
9432 return Register();
9433}
9434
9435Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9436 if (RetVT.SimpleTy != MVT::nxv8i1)
9437 return Register();
9438 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9439 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9440 }
9441 return Register();
9442}
9443
9444Register fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9445 if (RetVT.SimpleTy != MVT::nxv16i1)
9446 return Register();
9447 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9448 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9449 }
9450 return Register();
9451}
9452
9453Register fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9454 if (RetVT.SimpleTy != MVT::nxv16i8)
9455 return Register();
9456 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9457 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9458 }
9459 return Register();
9460}
9461
9462Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9463 if (RetVT.SimpleTy != MVT::nxv8i16)
9464 return Register();
9465 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9466 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9467 }
9468 return Register();
9469}
9470
9471Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9472 if (RetVT.SimpleTy != MVT::nxv4i32)
9473 return Register();
9474 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9475 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9476 }
9477 return Register();
9478}
9479
9480Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9481 if (RetVT.SimpleTy != MVT::nxv2i64)
9482 return Register();
9483 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9484 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9485 }
9486 return Register();
9487}
9488
9489Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9490 if (RetVT.SimpleTy != MVT::nxv2f16)
9491 return Register();
9492 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9493 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9494 }
9495 return Register();
9496}
9497
9498Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9499 if (RetVT.SimpleTy != MVT::nxv4f16)
9500 return Register();
9501 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9502 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9503 }
9504 return Register();
9505}
9506
9507Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9508 if (RetVT.SimpleTy != MVT::nxv8f16)
9509 return Register();
9510 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9511 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9512 }
9513 return Register();
9514}
9515
9516Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9517 if (RetVT.SimpleTy != MVT::nxv2bf16)
9518 return Register();
9519 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9520 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9521 }
9522 return Register();
9523}
9524
9525Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9526 if (RetVT.SimpleTy != MVT::nxv4bf16)
9527 return Register();
9528 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9529 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9530 }
9531 return Register();
9532}
9533
9534Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9535 if (RetVT.SimpleTy != MVT::nxv8bf16)
9536 return Register();
9537 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9538 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9539 }
9540 return Register();
9541}
9542
9543Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9544 if (RetVT.SimpleTy != MVT::nxv2f32)
9545 return Register();
9546 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9547 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9548 }
9549 return Register();
9550}
9551
9552Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9553 if (RetVT.SimpleTy != MVT::nxv4f32)
9554 return Register();
9555 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9556 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9557 }
9558 return Register();
9559}
9560
9561Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9562 if (RetVT.SimpleTy != MVT::nxv2f64)
9563 return Register();
9564 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9565 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9566 }
9567 return Register();
9568}
9569
9570Register fastEmit_AArch64ISD_ZIP1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9571 switch (VT.SimpleTy) {
9572 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(RetVT, Op0, Op1);
9573 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(RetVT, Op0, Op1);
9574 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(RetVT, Op0, Op1);
9575 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(RetVT, Op0, Op1);
9576 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(RetVT, Op0, Op1);
9577 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(RetVT, Op0, Op1);
9578 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(RetVT, Op0, Op1);
9579 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(RetVT, Op0, Op1);
9580 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(RetVT, Op0, Op1);
9581 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(RetVT, Op0, Op1);
9582 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(RetVT, Op0, Op1);
9583 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(RetVT, Op0, Op1);
9584 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(RetVT, Op0, Op1);
9585 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(RetVT, Op0, Op1);
9586 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9587 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9588 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9589 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9590 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9591 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9592 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9593 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9594 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9595 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9596 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9597 case MVT::nxv2bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9598 case MVT::nxv4bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9599 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9600 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9601 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9602 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9603 default: return Register();
9604 }
9605}
9606
9607// FastEmit functions for AArch64ISD::ZIP2.
9608
9609Register fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9610 if (RetVT.SimpleTy != MVT::v8i8)
9611 return Register();
9612 if ((Subtarget->isNeonAvailable())) {
9613 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9614 }
9615 return Register();
9616}
9617
9618Register fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9619 if (RetVT.SimpleTy != MVT::v16i8)
9620 return Register();
9621 if ((Subtarget->isNeonAvailable())) {
9622 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9623 }
9624 return Register();
9625}
9626
9627Register fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9628 if (RetVT.SimpleTy != MVT::v4i16)
9629 return Register();
9630 if ((Subtarget->isNeonAvailable())) {
9631 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9632 }
9633 return Register();
9634}
9635
9636Register fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9637 if (RetVT.SimpleTy != MVT::v8i16)
9638 return Register();
9639 if ((Subtarget->isNeonAvailable())) {
9640 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9641 }
9642 return Register();
9643}
9644
9645Register fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9646 if (RetVT.SimpleTy != MVT::v2i32)
9647 return Register();
9648 if ((Subtarget->isNeonAvailable())) {
9649 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9650 }
9651 return Register();
9652}
9653
9654Register fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9655 if (RetVT.SimpleTy != MVT::v4i32)
9656 return Register();
9657 if ((Subtarget->isNeonAvailable())) {
9658 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9659 }
9660 return Register();
9661}
9662
9663Register fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9664 if (RetVT.SimpleTy != MVT::v2i64)
9665 return Register();
9666 if ((Subtarget->isNeonAvailable())) {
9667 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9668 }
9669 return Register();
9670}
9671
9672Register fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9673 if (RetVT.SimpleTy != MVT::v4f16)
9674 return Register();
9675 if ((Subtarget->isNeonAvailable())) {
9676 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9677 }
9678 return Register();
9679}
9680
9681Register fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9682 if (RetVT.SimpleTy != MVT::v8f16)
9683 return Register();
9684 if ((Subtarget->isNeonAvailable())) {
9685 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9686 }
9687 return Register();
9688}
9689
9690Register fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9691 if (RetVT.SimpleTy != MVT::v4bf16)
9692 return Register();
9693 if ((Subtarget->isNeonAvailable())) {
9694 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9695 }
9696 return Register();
9697}
9698
9699Register fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9700 if (RetVT.SimpleTy != MVT::v8bf16)
9701 return Register();
9702 if ((Subtarget->isNeonAvailable())) {
9703 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9704 }
9705 return Register();
9706}
9707
9708Register fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9709 if (RetVT.SimpleTy != MVT::v2f32)
9710 return Register();
9711 if ((Subtarget->isNeonAvailable())) {
9712 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9713 }
9714 return Register();
9715}
9716
9717Register fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9718 if (RetVT.SimpleTy != MVT::v4f32)
9719 return Register();
9720 if ((Subtarget->isNeonAvailable())) {
9721 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9722 }
9723 return Register();
9724}
9725
9726Register fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9727 if (RetVT.SimpleTy != MVT::v2f64)
9728 return Register();
9729 if ((Subtarget->isNeonAvailable())) {
9730 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9731 }
9732 return Register();
9733}
9734
9735Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9736 if (RetVT.SimpleTy != MVT::nxv2i1)
9737 return Register();
9738 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9739 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9740 }
9741 return Register();
9742}
9743
9744Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9745 if (RetVT.SimpleTy != MVT::nxv4i1)
9746 return Register();
9747 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9748 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9749 }
9750 return Register();
9751}
9752
9753Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9754 if (RetVT.SimpleTy != MVT::nxv8i1)
9755 return Register();
9756 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9757 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9758 }
9759 return Register();
9760}
9761
9762Register fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9763 if (RetVT.SimpleTy != MVT::nxv16i1)
9764 return Register();
9765 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9766 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9767 }
9768 return Register();
9769}
9770
9771Register fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9772 if (RetVT.SimpleTy != MVT::nxv16i8)
9773 return Register();
9774 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9775 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9776 }
9777 return Register();
9778}
9779
9780Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9781 if (RetVT.SimpleTy != MVT::nxv8i16)
9782 return Register();
9783 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9784 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9785 }
9786 return Register();
9787}
9788
9789Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9790 if (RetVT.SimpleTy != MVT::nxv4i32)
9791 return Register();
9792 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9793 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9794 }
9795 return Register();
9796}
9797
9798Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9799 if (RetVT.SimpleTy != MVT::nxv2i64)
9800 return Register();
9801 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9802 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9803 }
9804 return Register();
9805}
9806
9807Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9808 if (RetVT.SimpleTy != MVT::nxv2f16)
9809 return Register();
9810 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9811 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9812 }
9813 return Register();
9814}
9815
9816Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9817 if (RetVT.SimpleTy != MVT::nxv4f16)
9818 return Register();
9819 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9820 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9821 }
9822 return Register();
9823}
9824
9825Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9826 if (RetVT.SimpleTy != MVT::nxv8f16)
9827 return Register();
9828 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9829 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9830 }
9831 return Register();
9832}
9833
9834Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9835 if (RetVT.SimpleTy != MVT::nxv2bf16)
9836 return Register();
9837 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9838 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9839 }
9840 return Register();
9841}
9842
9843Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9844 if (RetVT.SimpleTy != MVT::nxv4bf16)
9845 return Register();
9846 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9847 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9848 }
9849 return Register();
9850}
9851
9852Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9853 if (RetVT.SimpleTy != MVT::nxv8bf16)
9854 return Register();
9855 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9856 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9857 }
9858 return Register();
9859}
9860
9861Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9862 if (RetVT.SimpleTy != MVT::nxv2f32)
9863 return Register();
9864 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9865 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9866 }
9867 return Register();
9868}
9869
9870Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9871 if (RetVT.SimpleTy != MVT::nxv4f32)
9872 return Register();
9873 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9874 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9875 }
9876 return Register();
9877}
9878
9879Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9880 if (RetVT.SimpleTy != MVT::nxv2f64)
9881 return Register();
9882 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9883 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9884 }
9885 return Register();
9886}
9887
9888Register fastEmit_AArch64ISD_ZIP2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9889 switch (VT.SimpleTy) {
9890 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(RetVT, Op0, Op1);
9891 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(RetVT, Op0, Op1);
9892 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(RetVT, Op0, Op1);
9893 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(RetVT, Op0, Op1);
9894 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(RetVT, Op0, Op1);
9895 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(RetVT, Op0, Op1);
9896 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(RetVT, Op0, Op1);
9897 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(RetVT, Op0, Op1);
9898 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(RetVT, Op0, Op1);
9899 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(RetVT, Op0, Op1);
9900 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(RetVT, Op0, Op1);
9901 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(RetVT, Op0, Op1);
9902 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(RetVT, Op0, Op1);
9903 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(RetVT, Op0, Op1);
9904 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9905 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9906 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9907 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9908 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9909 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9910 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9911 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9912 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9913 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9914 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9915 case MVT::nxv2bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9916 case MVT::nxv4bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9917 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9918 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9919 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9920 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9921 default: return Register();
9922 }
9923}
9924
9925// FastEmit functions for ISD::ABDS.
9926
9927Register fastEmit_ISD_ABDS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9928 if (RetVT.SimpleTy != MVT::v8i8)
9929 return Register();
9930 if ((Subtarget->isNeonAvailable())) {
9931 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9932 }
9933 return Register();
9934}
9935
9936Register fastEmit_ISD_ABDS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9937 if (RetVT.SimpleTy != MVT::v16i8)
9938 return Register();
9939 if ((Subtarget->isNeonAvailable())) {
9940 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9941 }
9942 return Register();
9943}
9944
9945Register fastEmit_ISD_ABDS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9946 if (RetVT.SimpleTy != MVT::v4i16)
9947 return Register();
9948 if ((Subtarget->isNeonAvailable())) {
9949 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9950 }
9951 return Register();
9952}
9953
9954Register fastEmit_ISD_ABDS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9955 if (RetVT.SimpleTy != MVT::v8i16)
9956 return Register();
9957 if ((Subtarget->isNeonAvailable())) {
9958 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9959 }
9960 return Register();
9961}
9962
9963Register fastEmit_ISD_ABDS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9964 if (RetVT.SimpleTy != MVT::v2i32)
9965 return Register();
9966 if ((Subtarget->isNeonAvailable())) {
9967 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9968 }
9969 return Register();
9970}
9971
9972Register fastEmit_ISD_ABDS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9973 if (RetVT.SimpleTy != MVT::v4i32)
9974 return Register();
9975 if ((Subtarget->isNeonAvailable())) {
9976 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9977 }
9978 return Register();
9979}
9980
9981Register fastEmit_ISD_ABDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9982 switch (VT.SimpleTy) {
9983 case MVT::v8i8: return fastEmit_ISD_ABDS_MVT_v8i8_rr(RetVT, Op0, Op1);
9984 case MVT::v16i8: return fastEmit_ISD_ABDS_MVT_v16i8_rr(RetVT, Op0, Op1);
9985 case MVT::v4i16: return fastEmit_ISD_ABDS_MVT_v4i16_rr(RetVT, Op0, Op1);
9986 case MVT::v8i16: return fastEmit_ISD_ABDS_MVT_v8i16_rr(RetVT, Op0, Op1);
9987 case MVT::v2i32: return fastEmit_ISD_ABDS_MVT_v2i32_rr(RetVT, Op0, Op1);
9988 case MVT::v4i32: return fastEmit_ISD_ABDS_MVT_v4i32_rr(RetVT, Op0, Op1);
9989 default: return Register();
9990 }
9991}
9992
9993// FastEmit functions for ISD::ABDU.
9994
9995Register fastEmit_ISD_ABDU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9996 if (RetVT.SimpleTy != MVT::v8i8)
9997 return Register();
9998 if ((Subtarget->isNeonAvailable())) {
9999 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10000 }
10001 return Register();
10002}
10003
10004Register fastEmit_ISD_ABDU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10005 if (RetVT.SimpleTy != MVT::v16i8)
10006 return Register();
10007 if ((Subtarget->isNeonAvailable())) {
10008 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10009 }
10010 return Register();
10011}
10012
10013Register fastEmit_ISD_ABDU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10014 if (RetVT.SimpleTy != MVT::v4i16)
10015 return Register();
10016 if ((Subtarget->isNeonAvailable())) {
10017 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10018 }
10019 return Register();
10020}
10021
10022Register fastEmit_ISD_ABDU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10023 if (RetVT.SimpleTy != MVT::v8i16)
10024 return Register();
10025 if ((Subtarget->isNeonAvailable())) {
10026 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10027 }
10028 return Register();
10029}
10030
10031Register fastEmit_ISD_ABDU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10032 if (RetVT.SimpleTy != MVT::v2i32)
10033 return Register();
10034 if ((Subtarget->isNeonAvailable())) {
10035 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10036 }
10037 return Register();
10038}
10039
10040Register fastEmit_ISD_ABDU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10041 if (RetVT.SimpleTy != MVT::v4i32)
10042 return Register();
10043 if ((Subtarget->isNeonAvailable())) {
10044 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10045 }
10046 return Register();
10047}
10048
10049Register fastEmit_ISD_ABDU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10050 switch (VT.SimpleTy) {
10051 case MVT::v8i8: return fastEmit_ISD_ABDU_MVT_v8i8_rr(RetVT, Op0, Op1);
10052 case MVT::v16i8: return fastEmit_ISD_ABDU_MVT_v16i8_rr(RetVT, Op0, Op1);
10053 case MVT::v4i16: return fastEmit_ISD_ABDU_MVT_v4i16_rr(RetVT, Op0, Op1);
10054 case MVT::v8i16: return fastEmit_ISD_ABDU_MVT_v8i16_rr(RetVT, Op0, Op1);
10055 case MVT::v2i32: return fastEmit_ISD_ABDU_MVT_v2i32_rr(RetVT, Op0, Op1);
10056 case MVT::v4i32: return fastEmit_ISD_ABDU_MVT_v4i32_rr(RetVT, Op0, Op1);
10057 default: return Register();
10058 }
10059}
10060
10061// FastEmit functions for ISD::ADD.
10062
10063Register fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10064 if (RetVT.SimpleTy != MVT::i32)
10065 return Register();
10066 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
10067}
10068
10069Register fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10070 if (RetVT.SimpleTy != MVT::i64)
10071 return Register();
10072 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
10073}
10074
10075Register fastEmit_ISD_ADD_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10076 if (RetVT.SimpleTy != MVT::v8i8)
10077 return Register();
10078 if ((Subtarget->isNeonAvailable())) {
10079 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10080 }
10081 return Register();
10082}
10083
10084Register fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10085 if (RetVT.SimpleTy != MVT::v16i8)
10086 return Register();
10087 if ((Subtarget->isNeonAvailable())) {
10088 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10089 }
10090 return Register();
10091}
10092
10093Register fastEmit_ISD_ADD_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10094 if (RetVT.SimpleTy != MVT::v4i16)
10095 return Register();
10096 if ((Subtarget->isNeonAvailable())) {
10097 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10098 }
10099 return Register();
10100}
10101
10102Register fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10103 if (RetVT.SimpleTy != MVT::v8i16)
10104 return Register();
10105 if ((Subtarget->isNeonAvailable())) {
10106 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10107 }
10108 return Register();
10109}
10110
10111Register fastEmit_ISD_ADD_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10112 if (RetVT.SimpleTy != MVT::v2i32)
10113 return Register();
10114 if ((Subtarget->isNeonAvailable())) {
10115 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10116 }
10117 return Register();
10118}
10119
10120Register fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10121 if (RetVT.SimpleTy != MVT::v4i32)
10122 return Register();
10123 if ((Subtarget->isNeonAvailable())) {
10124 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10125 }
10126 return Register();
10127}
10128
10129Register fastEmit_ISD_ADD_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
10130 if (RetVT.SimpleTy != MVT::v1i64)
10131 return Register();
10132 if ((Subtarget->isNeonAvailable())) {
10133 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
10134 }
10135 return Register();
10136}
10137
10138Register fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10139 if (RetVT.SimpleTy != MVT::v2i64)
10140 return Register();
10141 if ((Subtarget->isNeonAvailable())) {
10142 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10143 }
10144 return Register();
10145}
10146
10147Register fastEmit_ISD_ADD_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10148 if (RetVT.SimpleTy != MVT::nxv16i8)
10149 return Register();
10150 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10151 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
10152 }
10153 return Register();
10154}
10155
10156Register fastEmit_ISD_ADD_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10157 if (RetVT.SimpleTy != MVT::nxv8i16)
10158 return Register();
10159 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10160 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10161 }
10162 return Register();
10163}
10164
10165Register fastEmit_ISD_ADD_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10166 if (RetVT.SimpleTy != MVT::nxv4i32)
10167 return Register();
10168 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10169 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10170 }
10171 return Register();
10172}
10173
10174Register fastEmit_ISD_ADD_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10175 if (RetVT.SimpleTy != MVT::nxv2i64)
10176 return Register();
10177 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10178 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10179 }
10180 return Register();
10181}
10182
10183Register fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10184 switch (VT.SimpleTy) {
10185 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
10186 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op1);
10187 case MVT::v8i8: return fastEmit_ISD_ADD_MVT_v8i8_rr(RetVT, Op0, Op1);
10188 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
10189 case MVT::v4i16: return fastEmit_ISD_ADD_MVT_v4i16_rr(RetVT, Op0, Op1);
10190 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
10191 case MVT::v2i32: return fastEmit_ISD_ADD_MVT_v2i32_rr(RetVT, Op0, Op1);
10192 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
10193 case MVT::v1i64: return fastEmit_ISD_ADD_MVT_v1i64_rr(RetVT, Op0, Op1);
10194 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
10195 case MVT::nxv16i8: return fastEmit_ISD_ADD_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10196 case MVT::nxv8i16: return fastEmit_ISD_ADD_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10197 case MVT::nxv4i32: return fastEmit_ISD_ADD_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10198 case MVT::nxv2i64: return fastEmit_ISD_ADD_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10199 default: return Register();
10200 }
10201}
10202
10203// FastEmit functions for ISD::AND.
10204
10205Register fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10206 if (RetVT.SimpleTy != MVT::i32)
10207 return Register();
10208 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
10209}
10210
10211Register fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10212 if (RetVT.SimpleTy != MVT::i64)
10213 return Register();
10214 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
10215}
10216
10217Register fastEmit_ISD_AND_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10218 if (RetVT.SimpleTy != MVT::v8i8)
10219 return Register();
10220 if ((Subtarget->isNeonAvailable())) {
10221 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10222 }
10223 return Register();
10224}
10225
10226Register fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10227 if (RetVT.SimpleTy != MVT::v16i8)
10228 return Register();
10229 if ((Subtarget->isNeonAvailable())) {
10230 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10231 }
10232 return Register();
10233}
10234
10235Register fastEmit_ISD_AND_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10236 if (RetVT.SimpleTy != MVT::v4i16)
10237 return Register();
10238 if ((Subtarget->isNeonAvailable())) {
10239 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10240 }
10241 return Register();
10242}
10243
10244Register fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10245 if (RetVT.SimpleTy != MVT::v8i16)
10246 return Register();
10247 if ((Subtarget->isNeonAvailable())) {
10248 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10249 }
10250 return Register();
10251}
10252
10253Register fastEmit_ISD_AND_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10254 if (RetVT.SimpleTy != MVT::v2i32)
10255 return Register();
10256 if ((Subtarget->isNeonAvailable())) {
10257 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10258 }
10259 return Register();
10260}
10261
10262Register fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10263 if (RetVT.SimpleTy != MVT::v4i32)
10264 return Register();
10265 if ((Subtarget->isNeonAvailable())) {
10266 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10267 }
10268 return Register();
10269}
10270
10271Register fastEmit_ISD_AND_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
10272 if (RetVT.SimpleTy != MVT::v1i64)
10273 return Register();
10274 if ((Subtarget->isNeonAvailable())) {
10275 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10276 }
10277 return Register();
10278}
10279
10280Register fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10281 if (RetVT.SimpleTy != MVT::v2i64)
10282 return Register();
10283 if ((Subtarget->isNeonAvailable())) {
10284 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10285 }
10286 return Register();
10287}
10288
10289Register fastEmit_ISD_AND_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10290 if (RetVT.SimpleTy != MVT::nxv16i8)
10291 return Register();
10292 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10293 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10294 }
10295 return Register();
10296}
10297
10298Register fastEmit_ISD_AND_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10299 if (RetVT.SimpleTy != MVT::nxv8i16)
10300 return Register();
10301 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10302 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10303 }
10304 return Register();
10305}
10306
10307Register fastEmit_ISD_AND_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10308 if (RetVT.SimpleTy != MVT::nxv4i32)
10309 return Register();
10310 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10311 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10312 }
10313 return Register();
10314}
10315
10316Register fastEmit_ISD_AND_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10317 if (RetVT.SimpleTy != MVT::nxv2i64)
10318 return Register();
10319 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10320 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10321 }
10322 return Register();
10323}
10324
10325Register fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10326 switch (VT.SimpleTy) {
10327 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
10328 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op1);
10329 case MVT::v8i8: return fastEmit_ISD_AND_MVT_v8i8_rr(RetVT, Op0, Op1);
10330 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
10331 case MVT::v4i16: return fastEmit_ISD_AND_MVT_v4i16_rr(RetVT, Op0, Op1);
10332 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
10333 case MVT::v2i32: return fastEmit_ISD_AND_MVT_v2i32_rr(RetVT, Op0, Op1);
10334 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
10335 case MVT::v1i64: return fastEmit_ISD_AND_MVT_v1i64_rr(RetVT, Op0, Op1);
10336 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
10337 case MVT::nxv16i8: return fastEmit_ISD_AND_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10338 case MVT::nxv8i16: return fastEmit_ISD_AND_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10339 case MVT::nxv4i32: return fastEmit_ISD_AND_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10340 case MVT::nxv2i64: return fastEmit_ISD_AND_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10341 default: return Register();
10342 }
10343}
10344
10345// FastEmit functions for ISD::AVGCEILS.
10346
10347Register fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10348 if (RetVT.SimpleTy != MVT::v8i8)
10349 return Register();
10350 if ((Subtarget->isNeonAvailable())) {
10351 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10352 }
10353 return Register();
10354}
10355
10356Register fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10357 if (RetVT.SimpleTy != MVT::v16i8)
10358 return Register();
10359 if ((Subtarget->isNeonAvailable())) {
10360 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10361 }
10362 return Register();
10363}
10364
10365Register fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10366 if (RetVT.SimpleTy != MVT::v4i16)
10367 return Register();
10368 if ((Subtarget->isNeonAvailable())) {
10369 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10370 }
10371 return Register();
10372}
10373
10374Register fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10375 if (RetVT.SimpleTy != MVT::v8i16)
10376 return Register();
10377 if ((Subtarget->isNeonAvailable())) {
10378 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10379 }
10380 return Register();
10381}
10382
10383Register fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10384 if (RetVT.SimpleTy != MVT::v2i32)
10385 return Register();
10386 if ((Subtarget->isNeonAvailable())) {
10387 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10388 }
10389 return Register();
10390}
10391
10392Register fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10393 if (RetVT.SimpleTy != MVT::v4i32)
10394 return Register();
10395 if ((Subtarget->isNeonAvailable())) {
10396 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10397 }
10398 return Register();
10399}
10400
10401Register fastEmit_ISD_AVGCEILS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10402 switch (VT.SimpleTy) {
10403 case MVT::v8i8: return fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(RetVT, Op0, Op1);
10404 case MVT::v16i8: return fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(RetVT, Op0, Op1);
10405 case MVT::v4i16: return fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(RetVT, Op0, Op1);
10406 case MVT::v8i16: return fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(RetVT, Op0, Op1);
10407 case MVT::v2i32: return fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(RetVT, Op0, Op1);
10408 case MVT::v4i32: return fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(RetVT, Op0, Op1);
10409 default: return Register();
10410 }
10411}
10412
10413// FastEmit functions for ISD::AVGCEILU.
10414
10415Register fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10416 if (RetVT.SimpleTy != MVT::v8i8)
10417 return Register();
10418 if ((Subtarget->isNeonAvailable())) {
10419 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10420 }
10421 return Register();
10422}
10423
10424Register fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10425 if (RetVT.SimpleTy != MVT::v16i8)
10426 return Register();
10427 if ((Subtarget->isNeonAvailable())) {
10428 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10429 }
10430 return Register();
10431}
10432
10433Register fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10434 if (RetVT.SimpleTy != MVT::v4i16)
10435 return Register();
10436 if ((Subtarget->isNeonAvailable())) {
10437 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10438 }
10439 return Register();
10440}
10441
10442Register fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10443 if (RetVT.SimpleTy != MVT::v8i16)
10444 return Register();
10445 if ((Subtarget->isNeonAvailable())) {
10446 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10447 }
10448 return Register();
10449}
10450
10451Register fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10452 if (RetVT.SimpleTy != MVT::v2i32)
10453 return Register();
10454 if ((Subtarget->isNeonAvailable())) {
10455 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10456 }
10457 return Register();
10458}
10459
10460Register fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10461 if (RetVT.SimpleTy != MVT::v4i32)
10462 return Register();
10463 if ((Subtarget->isNeonAvailable())) {
10464 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10465 }
10466 return Register();
10467}
10468
10469Register fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10470 switch (VT.SimpleTy) {
10471 case MVT::v8i8: return fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(RetVT, Op0, Op1);
10472 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
10473 case MVT::v4i16: return fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(RetVT, Op0, Op1);
10474 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
10475 case MVT::v2i32: return fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(RetVT, Op0, Op1);
10476 case MVT::v4i32: return fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(RetVT, Op0, Op1);
10477 default: return Register();
10478 }
10479}
10480
10481// FastEmit functions for ISD::AVGFLOORS.
10482
10483Register fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10484 if (RetVT.SimpleTy != MVT::v8i8)
10485 return Register();
10486 if ((Subtarget->isNeonAvailable())) {
10487 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10488 }
10489 return Register();
10490}
10491
10492Register fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10493 if (RetVT.SimpleTy != MVT::v16i8)
10494 return Register();
10495 if ((Subtarget->isNeonAvailable())) {
10496 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10497 }
10498 return Register();
10499}
10500
10501Register fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10502 if (RetVT.SimpleTy != MVT::v4i16)
10503 return Register();
10504 if ((Subtarget->isNeonAvailable())) {
10505 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10506 }
10507 return Register();
10508}
10509
10510Register fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10511 if (RetVT.SimpleTy != MVT::v8i16)
10512 return Register();
10513 if ((Subtarget->isNeonAvailable())) {
10514 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10515 }
10516 return Register();
10517}
10518
10519Register fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10520 if (RetVT.SimpleTy != MVT::v2i32)
10521 return Register();
10522 if ((Subtarget->isNeonAvailable())) {
10523 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10524 }
10525 return Register();
10526}
10527
10528Register fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10529 if (RetVT.SimpleTy != MVT::v4i32)
10530 return Register();
10531 if ((Subtarget->isNeonAvailable())) {
10532 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10533 }
10534 return Register();
10535}
10536
10537Register fastEmit_ISD_AVGFLOORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10538 switch (VT.SimpleTy) {
10539 case MVT::v8i8: return fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(RetVT, Op0, Op1);
10540 case MVT::v16i8: return fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(RetVT, Op0, Op1);
10541 case MVT::v4i16: return fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(RetVT, Op0, Op1);
10542 case MVT::v8i16: return fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(RetVT, Op0, Op1);
10543 case MVT::v2i32: return fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(RetVT, Op0, Op1);
10544 case MVT::v4i32: return fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(RetVT, Op0, Op1);
10545 default: return Register();
10546 }
10547}
10548
10549// FastEmit functions for ISD::AVGFLOORU.
10550
10551Register fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10552 if (RetVT.SimpleTy != MVT::v8i8)
10553 return Register();
10554 if ((Subtarget->isNeonAvailable())) {
10555 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10556 }
10557 return Register();
10558}
10559
10560Register fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10561 if (RetVT.SimpleTy != MVT::v16i8)
10562 return Register();
10563 if ((Subtarget->isNeonAvailable())) {
10564 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10565 }
10566 return Register();
10567}
10568
10569Register fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10570 if (RetVT.SimpleTy != MVT::v4i16)
10571 return Register();
10572 if ((Subtarget->isNeonAvailable())) {
10573 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10574 }
10575 return Register();
10576}
10577
10578Register fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10579 if (RetVT.SimpleTy != MVT::v8i16)
10580 return Register();
10581 if ((Subtarget->isNeonAvailable())) {
10582 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10583 }
10584 return Register();
10585}
10586
10587Register fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10588 if (RetVT.SimpleTy != MVT::v2i32)
10589 return Register();
10590 if ((Subtarget->isNeonAvailable())) {
10591 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10592 }
10593 return Register();
10594}
10595
10596Register fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10597 if (RetVT.SimpleTy != MVT::v4i32)
10598 return Register();
10599 if ((Subtarget->isNeonAvailable())) {
10600 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10601 }
10602 return Register();
10603}
10604
10605Register fastEmit_ISD_AVGFLOORU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10606 switch (VT.SimpleTy) {
10607 case MVT::v8i8: return fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(RetVT, Op0, Op1);
10608 case MVT::v16i8: return fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(RetVT, Op0, Op1);
10609 case MVT::v4i16: return fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(RetVT, Op0, Op1);
10610 case MVT::v8i16: return fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(RetVT, Op0, Op1);
10611 case MVT::v2i32: return fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(RetVT, Op0, Op1);
10612 case MVT::v4i32: return fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(RetVT, Op0, Op1);
10613 default: return Register();
10614 }
10615}
10616
10617// FastEmit functions for ISD::CLMUL.
10618
10619Register fastEmit_ISD_CLMUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10620 if (RetVT.SimpleTy != MVT::v8i8)
10621 return Register();
10622 if ((Subtarget->isNeonAvailable())) {
10623 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10624 }
10625 return Register();
10626}
10627
10628Register fastEmit_ISD_CLMUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10629 if (RetVT.SimpleTy != MVT::v16i8)
10630 return Register();
10631 if ((Subtarget->isNeonAvailable())) {
10632 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10633 }
10634 return Register();
10635}
10636
10637Register fastEmit_ISD_CLMUL_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10638 if (RetVT.SimpleTy != MVT::nxv16i8)
10639 return Register();
10640 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2() || Subtarget->hasSME()))) {
10641 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMUL_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
10642 }
10643 return Register();
10644}
10645
10646Register fastEmit_ISD_CLMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10647 switch (VT.SimpleTy) {
10648 case MVT::v8i8: return fastEmit_ISD_CLMUL_MVT_v8i8_rr(RetVT, Op0, Op1);
10649 case MVT::v16i8: return fastEmit_ISD_CLMUL_MVT_v16i8_rr(RetVT, Op0, Op1);
10650 case MVT::nxv16i8: return fastEmit_ISD_CLMUL_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10651 default: return Register();
10652 }
10653}
10654
10655// FastEmit functions for ISD::CONCAT_VECTORS.
10656
10657Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(MVT RetVT, Register Op0, Register Op1) {
10658 if (RetVT.SimpleTy != MVT::nxv2i1)
10659 return Register();
10660 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10661 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
10662 }
10663 return Register();
10664}
10665
10666Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
10667 if (RetVT.SimpleTy != MVT::nxv4i1)
10668 return Register();
10669 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10670 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
10671 }
10672 return Register();
10673}
10674
10675Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
10676 if (RetVT.SimpleTy != MVT::nxv8i1)
10677 return Register();
10678 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10679 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
10680 }
10681 return Register();
10682}
10683
10684Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
10685 if (RetVT.SimpleTy != MVT::nxv16i1)
10686 return Register();
10687 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10688 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
10689 }
10690 return Register();
10691}
10692
10693Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
10694 if (RetVT.SimpleTy != MVT::nxv4f16)
10695 return Register();
10696 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10697 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10698 }
10699 return Register();
10700}
10701
10702Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10703 if (RetVT.SimpleTy != MVT::nxv8f16)
10704 return Register();
10705 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10706 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10707 }
10708 return Register();
10709}
10710
10711Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10712 if (RetVT.SimpleTy != MVT::nxv4bf16)
10713 return Register();
10714 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10715 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10716 }
10717 return Register();
10718}
10719
10720Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10721 if (RetVT.SimpleTy != MVT::nxv8bf16)
10722 return Register();
10723 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10724 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10725 }
10726 return Register();
10727}
10728
10729Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10730 if (RetVT.SimpleTy != MVT::nxv4f32)
10731 return Register();
10732 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10733 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10734 }
10735 return Register();
10736}
10737
10738Register fastEmit_ISD_CONCAT_VECTORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10739 switch (VT.SimpleTy) {
10740 case MVT::nxv1i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(RetVT, Op0, Op1);
10741 case MVT::nxv2i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(RetVT, Op0, Op1);
10742 case MVT::nxv4i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(RetVT, Op0, Op1);
10743 case MVT::nxv8i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(RetVT, Op0, Op1);
10744 case MVT::nxv2f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(RetVT, Op0, Op1);
10745 case MVT::nxv4f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(RetVT, Op0, Op1);
10746 case MVT::nxv2bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
10747 case MVT::nxv4bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
10748 case MVT::nxv2f32: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(RetVT, Op0, Op1);
10749 default: return Register();
10750 }
10751}
10752
10753// FastEmit functions for ISD::FADD.
10754
10755Register fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10756 if (RetVT.SimpleTy != MVT::f16)
10757 return Register();
10758 if ((Subtarget->hasFullFP16())) {
10759 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
10760 }
10761 return Register();
10762}
10763
10764Register fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10765 if (RetVT.SimpleTy != MVT::f32)
10766 return Register();
10767 if ((Subtarget->hasFPARMv8())) {
10768 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
10769 }
10770 return Register();
10771}
10772
10773Register fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
10774 if (RetVT.SimpleTy != MVT::f64)
10775 return Register();
10776 if ((Subtarget->hasFPARMv8())) {
10777 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10778 }
10779 return Register();
10780}
10781
10782Register fastEmit_ISD_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10783 if (RetVT.SimpleTy != MVT::v4f16)
10784 return Register();
10785 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10786 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10787 }
10788 return Register();
10789}
10790
10791Register fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10792 if (RetVT.SimpleTy != MVT::v8f16)
10793 return Register();
10794 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10795 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10796 }
10797 return Register();
10798}
10799
10800Register fastEmit_ISD_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10801 if (RetVT.SimpleTy != MVT::v2f32)
10802 return Register();
10803 if ((Subtarget->isNeonAvailable())) {
10804 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10805 }
10806 return Register();
10807}
10808
10809Register fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10810 if (RetVT.SimpleTy != MVT::v4f32)
10811 return Register();
10812 if ((Subtarget->isNeonAvailable())) {
10813 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10814 }
10815 return Register();
10816}
10817
10818Register fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10819 if (RetVT.SimpleTy != MVT::v2f64)
10820 return Register();
10821 if ((Subtarget->isNeonAvailable())) {
10822 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10823 }
10824 return Register();
10825}
10826
10827Register fastEmit_ISD_FADD_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10828 if (RetVT.SimpleTy != MVT::nxv8f16)
10829 return Register();
10830 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10831 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10832 }
10833 return Register();
10834}
10835
10836Register fastEmit_ISD_FADD_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10837 if (RetVT.SimpleTy != MVT::nxv8bf16)
10838 return Register();
10839 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
10840 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFADD_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10841 }
10842 return Register();
10843}
10844
10845Register fastEmit_ISD_FADD_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10846 if (RetVT.SimpleTy != MVT::nxv4f32)
10847 return Register();
10848 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10849 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10850 }
10851 return Register();
10852}
10853
10854Register fastEmit_ISD_FADD_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10855 if (RetVT.SimpleTy != MVT::nxv2f64)
10856 return Register();
10857 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10858 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10859 }
10860 return Register();
10861}
10862
10863Register fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10864 switch (VT.SimpleTy) {
10865 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
10866 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
10867 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
10868 case MVT::v4f16: return fastEmit_ISD_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
10869 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
10870 case MVT::v2f32: return fastEmit_ISD_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
10871 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
10872 case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
10873 case MVT::nxv8f16: return fastEmit_ISD_FADD_MVT_nxv8f16_rr(RetVT, Op0, Op1);
10874 case MVT::nxv8bf16: return fastEmit_ISD_FADD_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
10875 case MVT::nxv4f32: return fastEmit_ISD_FADD_MVT_nxv4f32_rr(RetVT, Op0, Op1);
10876 case MVT::nxv2f64: return fastEmit_ISD_FADD_MVT_nxv2f64_rr(RetVT, Op0, Op1);
10877 default: return Register();
10878 }
10879}
10880
10881// FastEmit functions for ISD::FDIV.
10882
10883Register fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10884 if (RetVT.SimpleTy != MVT::f16)
10885 return Register();
10886 if ((Subtarget->hasFullFP16())) {
10887 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
10888 }
10889 return Register();
10890}
10891
10892Register fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10893 if (RetVT.SimpleTy != MVT::f32)
10894 return Register();
10895 if ((Subtarget->hasFPARMv8())) {
10896 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
10897 }
10898 return Register();
10899}
10900
10901Register fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
10902 if (RetVT.SimpleTy != MVT::f64)
10903 return Register();
10904 if ((Subtarget->hasFPARMv8())) {
10905 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10906 }
10907 return Register();
10908}
10909
10910Register fastEmit_ISD_FDIV_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10911 if (RetVT.SimpleTy != MVT::v4f16)
10912 return Register();
10913 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10914 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10915 }
10916 return Register();
10917}
10918
10919Register fastEmit_ISD_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10920 if (RetVT.SimpleTy != MVT::v8f16)
10921 return Register();
10922 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10923 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10924 }
10925 return Register();
10926}
10927
10928Register fastEmit_ISD_FDIV_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10929 if (RetVT.SimpleTy != MVT::v2f32)
10930 return Register();
10931 if ((Subtarget->isNeonAvailable())) {
10932 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10933 }
10934 return Register();
10935}
10936
10937Register fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10938 if (RetVT.SimpleTy != MVT::v4f32)
10939 return Register();
10940 if ((Subtarget->isNeonAvailable())) {
10941 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10942 }
10943 return Register();
10944}
10945
10946Register fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10947 if (RetVT.SimpleTy != MVT::v2f64)
10948 return Register();
10949 if ((Subtarget->isNeonAvailable())) {
10950 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10951 }
10952 return Register();
10953}
10954
10955Register fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10956 switch (VT.SimpleTy) {
10957 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
10958 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
10959 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
10960 case MVT::v4f16: return fastEmit_ISD_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1);
10961 case MVT::v8f16: return fastEmit_ISD_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
10962 case MVT::v2f32: return fastEmit_ISD_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1);
10963 case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
10964 case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
10965 default: return Register();
10966 }
10967}
10968
10969// FastEmit functions for ISD::FMAXIMUM.
10970
10971Register fastEmit_ISD_FMAXIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10972 if (RetVT.SimpleTy != MVT::f16)
10973 return Register();
10974 if ((Subtarget->hasFullFP16())) {
10975 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
10976 }
10977 return Register();
10978}
10979
10980Register fastEmit_ISD_FMAXIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10981 if (RetVT.SimpleTy != MVT::f32)
10982 return Register();
10983 if ((Subtarget->hasFPARMv8())) {
10984 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
10985 }
10986 return Register();
10987}
10988
10989Register fastEmit_ISD_FMAXIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
10990 if (RetVT.SimpleTy != MVT::f64)
10991 return Register();
10992 if ((Subtarget->hasFPARMv8())) {
10993 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10994 }
10995 return Register();
10996}
10997
10998Register fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10999 if (RetVT.SimpleTy != MVT::v4f16)
11000 return Register();
11001 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11002 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11003 }
11004 return Register();
11005}
11006
11007Register fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11008 if (RetVT.SimpleTy != MVT::v8f16)
11009 return Register();
11010 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11011 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11012 }
11013 return Register();
11014}
11015
11016Register fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11017 if (RetVT.SimpleTy != MVT::v2f32)
11018 return Register();
11019 if ((Subtarget->isNeonAvailable())) {
11020 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11021 }
11022 return Register();
11023}
11024
11025Register fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11026 if (RetVT.SimpleTy != MVT::v4f32)
11027 return Register();
11028 if ((Subtarget->isNeonAvailable())) {
11029 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11030 }
11031 return Register();
11032}
11033
11034Register fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11035 if (RetVT.SimpleTy != MVT::v1f64)
11036 return Register();
11037 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11038}
11039
11040Register fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11041 if (RetVT.SimpleTy != MVT::v2f64)
11042 return Register();
11043 if ((Subtarget->isNeonAvailable())) {
11044 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11045 }
11046 return Register();
11047}
11048
11049Register fastEmit_ISD_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11050 switch (VT.SimpleTy) {
11051 case MVT::f16: return fastEmit_ISD_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1);
11052 case MVT::f32: return fastEmit_ISD_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1);
11053 case MVT::f64: return fastEmit_ISD_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1);
11054 case MVT::v4f16: return fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11055 case MVT::v8f16: return fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11056 case MVT::v2f32: return fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11057 case MVT::v4f32: return fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11058 case MVT::v1f64: return fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11059 case MVT::v2f64: return fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11060 default: return Register();
11061 }
11062}
11063
11064// FastEmit functions for ISD::FMAXNUM.
11065
11066Register fastEmit_ISD_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11067 if (RetVT.SimpleTy != MVT::f16)
11068 return Register();
11069 if ((Subtarget->hasFullFP16())) {
11070 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11071 }
11072 return Register();
11073}
11074
11075Register fastEmit_ISD_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11076 if (RetVT.SimpleTy != MVT::f32)
11077 return Register();
11078 if ((Subtarget->hasFPARMv8())) {
11079 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11080 }
11081 return Register();
11082}
11083
11084Register fastEmit_ISD_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11085 if (RetVT.SimpleTy != MVT::f64)
11086 return Register();
11087 if ((Subtarget->hasFPARMv8())) {
11088 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11089 }
11090 return Register();
11091}
11092
11093Register fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11094 if (RetVT.SimpleTy != MVT::v4f16)
11095 return Register();
11096 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11097 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11098 }
11099 return Register();
11100}
11101
11102Register fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11103 if (RetVT.SimpleTy != MVT::v8f16)
11104 return Register();
11105 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11106 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11107 }
11108 return Register();
11109}
11110
11111Register fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11112 if (RetVT.SimpleTy != MVT::v2f32)
11113 return Register();
11114 if ((Subtarget->isNeonAvailable())) {
11115 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11116 }
11117 return Register();
11118}
11119
11120Register fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11121 if (RetVT.SimpleTy != MVT::v4f32)
11122 return Register();
11123 if ((Subtarget->isNeonAvailable())) {
11124 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11125 }
11126 return Register();
11127}
11128
11129Register fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11130 if (RetVT.SimpleTy != MVT::v1f64)
11131 return Register();
11132 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11133}
11134
11135Register fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11136 if (RetVT.SimpleTy != MVT::v2f64)
11137 return Register();
11138 if ((Subtarget->isNeonAvailable())) {
11139 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11140 }
11141 return Register();
11142}
11143
11144Register fastEmit_ISD_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11145 switch (VT.SimpleTy) {
11146 case MVT::f16: return fastEmit_ISD_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
11147 case MVT::f32: return fastEmit_ISD_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
11148 case MVT::f64: return fastEmit_ISD_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
11149 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11150 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11151 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11152 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11153 case MVT::v1f64: return fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11154 case MVT::v2f64: return fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11155 default: return Register();
11156 }
11157}
11158
11159// FastEmit functions for ISD::FMAXNUM_IEEE.
11160
11161Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11162 if (RetVT.SimpleTy != MVT::f16)
11163 return Register();
11164 if ((Subtarget->hasFullFP16())) {
11165 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11166 }
11167 return Register();
11168}
11169
11170Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11171 if (RetVT.SimpleTy != MVT::f32)
11172 return Register();
11173 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11174}
11175
11176Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11177 if (RetVT.SimpleTy != MVT::f64)
11178 return Register();
11179 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11180}
11181
11182Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11183 if (RetVT.SimpleTy != MVT::v4f16)
11184 return Register();
11185 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11186 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11187 }
11188 return Register();
11189}
11190
11191Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11192 if (RetVT.SimpleTy != MVT::v8f16)
11193 return Register();
11194 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11195 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11196 }
11197 return Register();
11198}
11199
11200Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11201 if (RetVT.SimpleTy != MVT::v2f32)
11202 return Register();
11203 if ((Subtarget->isNeonAvailable())) {
11204 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11205 }
11206 return Register();
11207}
11208
11209Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11210 if (RetVT.SimpleTy != MVT::v4f32)
11211 return Register();
11212 if ((Subtarget->isNeonAvailable())) {
11213 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11214 }
11215 return Register();
11216}
11217
11218Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11219 if (RetVT.SimpleTy != MVT::v2f64)
11220 return Register();
11221 if ((Subtarget->isNeonAvailable())) {
11222 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11223 }
11224 return Register();
11225}
11226
11227Register fastEmit_ISD_FMAXNUM_IEEE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11228 switch (VT.SimpleTy) {
11229 case MVT::f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f16_rr(RetVT, Op0, Op1);
11230 case MVT::f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f32_rr(RetVT, Op0, Op1);
11231 case MVT::f64: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f64_rr(RetVT, Op0, Op1);
11232 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f16_rr(RetVT, Op0, Op1);
11233 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v8f16_rr(RetVT, Op0, Op1);
11234 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f32_rr(RetVT, Op0, Op1);
11235 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f32_rr(RetVT, Op0, Op1);
11236 case MVT::v2f64: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f64_rr(RetVT, Op0, Op1);
11237 default: return Register();
11238 }
11239}
11240
11241// FastEmit functions for ISD::FMINIMUM.
11242
11243Register fastEmit_ISD_FMINIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11244 if (RetVT.SimpleTy != MVT::f16)
11245 return Register();
11246 if ((Subtarget->hasFullFP16())) {
11247 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11248 }
11249 return Register();
11250}
11251
11252Register fastEmit_ISD_FMINIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11253 if (RetVT.SimpleTy != MVT::f32)
11254 return Register();
11255 if ((Subtarget->hasFPARMv8())) {
11256 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11257 }
11258 return Register();
11259}
11260
11261Register fastEmit_ISD_FMINIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11262 if (RetVT.SimpleTy != MVT::f64)
11263 return Register();
11264 if ((Subtarget->hasFPARMv8())) {
11265 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11266 }
11267 return Register();
11268}
11269
11270Register fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11271 if (RetVT.SimpleTy != MVT::v4f16)
11272 return Register();
11273 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11274 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11275 }
11276 return Register();
11277}
11278
11279Register fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11280 if (RetVT.SimpleTy != MVT::v8f16)
11281 return Register();
11282 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11283 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11284 }
11285 return Register();
11286}
11287
11288Register fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11289 if (RetVT.SimpleTy != MVT::v2f32)
11290 return Register();
11291 if ((Subtarget->isNeonAvailable())) {
11292 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11293 }
11294 return Register();
11295}
11296
11297Register fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11298 if (RetVT.SimpleTy != MVT::v4f32)
11299 return Register();
11300 if ((Subtarget->isNeonAvailable())) {
11301 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11302 }
11303 return Register();
11304}
11305
11306Register fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11307 if (RetVT.SimpleTy != MVT::v1f64)
11308 return Register();
11309 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11310}
11311
11312Register fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11313 if (RetVT.SimpleTy != MVT::v2f64)
11314 return Register();
11315 if ((Subtarget->isNeonAvailable())) {
11316 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11317 }
11318 return Register();
11319}
11320
11321Register fastEmit_ISD_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11322 switch (VT.SimpleTy) {
11323 case MVT::f16: return fastEmit_ISD_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1);
11324 case MVT::f32: return fastEmit_ISD_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1);
11325 case MVT::f64: return fastEmit_ISD_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1);
11326 case MVT::v4f16: return fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11327 case MVT::v8f16: return fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11328 case MVT::v2f32: return fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11329 case MVT::v4f32: return fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11330 case MVT::v1f64: return fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11331 case MVT::v2f64: return fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11332 default: return Register();
11333 }
11334}
11335
11336// FastEmit functions for ISD::FMINNUM.
11337
11338Register fastEmit_ISD_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11339 if (RetVT.SimpleTy != MVT::f16)
11340 return Register();
11341 if ((Subtarget->hasFullFP16())) {
11342 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11343 }
11344 return Register();
11345}
11346
11347Register fastEmit_ISD_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11348 if (RetVT.SimpleTy != MVT::f32)
11349 return Register();
11350 if ((Subtarget->hasFPARMv8())) {
11351 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11352 }
11353 return Register();
11354}
11355
11356Register fastEmit_ISD_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11357 if (RetVT.SimpleTy != MVT::f64)
11358 return Register();
11359 if ((Subtarget->hasFPARMv8())) {
11360 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11361 }
11362 return Register();
11363}
11364
11365Register fastEmit_ISD_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11366 if (RetVT.SimpleTy != MVT::v4f16)
11367 return Register();
11368 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11369 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11370 }
11371 return Register();
11372}
11373
11374Register fastEmit_ISD_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11375 if (RetVT.SimpleTy != MVT::v8f16)
11376 return Register();
11377 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11378 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11379 }
11380 return Register();
11381}
11382
11383Register fastEmit_ISD_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11384 if (RetVT.SimpleTy != MVT::v2f32)
11385 return Register();
11386 if ((Subtarget->isNeonAvailable())) {
11387 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11388 }
11389 return Register();
11390}
11391
11392Register fastEmit_ISD_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11393 if (RetVT.SimpleTy != MVT::v4f32)
11394 return Register();
11395 if ((Subtarget->isNeonAvailable())) {
11396 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11397 }
11398 return Register();
11399}
11400
11401Register fastEmit_ISD_FMINNUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11402 if (RetVT.SimpleTy != MVT::v1f64)
11403 return Register();
11404 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11405}
11406
11407Register fastEmit_ISD_FMINNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11408 if (RetVT.SimpleTy != MVT::v2f64)
11409 return Register();
11410 if ((Subtarget->isNeonAvailable())) {
11411 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11412 }
11413 return Register();
11414}
11415
11416Register fastEmit_ISD_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11417 switch (VT.SimpleTy) {
11418 case MVT::f16: return fastEmit_ISD_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
11419 case MVT::f32: return fastEmit_ISD_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
11420 case MVT::f64: return fastEmit_ISD_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
11421 case MVT::v4f16: return fastEmit_ISD_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11422 case MVT::v8f16: return fastEmit_ISD_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11423 case MVT::v2f32: return fastEmit_ISD_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11424 case MVT::v4f32: return fastEmit_ISD_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11425 case MVT::v1f64: return fastEmit_ISD_FMINNUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11426 case MVT::v2f64: return fastEmit_ISD_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11427 default: return Register();
11428 }
11429}
11430
11431// FastEmit functions for ISD::FMINNUM_IEEE.
11432
11433Register fastEmit_ISD_FMINNUM_IEEE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11434 if (RetVT.SimpleTy != MVT::f16)
11435 return Register();
11436 if ((Subtarget->hasFullFP16())) {
11437 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11438 }
11439 return Register();
11440}
11441
11442Register fastEmit_ISD_FMINNUM_IEEE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11443 if (RetVT.SimpleTy != MVT::f32)
11444 return Register();
11445 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11446}
11447
11448Register fastEmit_ISD_FMINNUM_IEEE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11449 if (RetVT.SimpleTy != MVT::f64)
11450 return Register();
11451 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11452}
11453
11454Register fastEmit_ISD_FMINNUM_IEEE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11455 if (RetVT.SimpleTy != MVT::v4f16)
11456 return Register();
11457 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11458 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11459 }
11460 return Register();
11461}
11462
11463Register fastEmit_ISD_FMINNUM_IEEE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11464 if (RetVT.SimpleTy != MVT::v8f16)
11465 return Register();
11466 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11467 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11468 }
11469 return Register();
11470}
11471
11472Register fastEmit_ISD_FMINNUM_IEEE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11473 if (RetVT.SimpleTy != MVT::v2f32)
11474 return Register();
11475 if ((Subtarget->isNeonAvailable())) {
11476 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11477 }
11478 return Register();
11479}
11480
11481Register fastEmit_ISD_FMINNUM_IEEE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11482 if (RetVT.SimpleTy != MVT::v4f32)
11483 return Register();
11484 if ((Subtarget->isNeonAvailable())) {
11485 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11486 }
11487 return Register();
11488}
11489
11490Register fastEmit_ISD_FMINNUM_IEEE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11491 if (RetVT.SimpleTy != MVT::v2f64)
11492 return Register();
11493 if ((Subtarget->isNeonAvailable())) {
11494 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11495 }
11496 return Register();
11497}
11498
11499Register fastEmit_ISD_FMINNUM_IEEE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11500 switch (VT.SimpleTy) {
11501 case MVT::f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_f16_rr(RetVT, Op0, Op1);
11502 case MVT::f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_f32_rr(RetVT, Op0, Op1);
11503 case MVT::f64: return fastEmit_ISD_FMINNUM_IEEE_MVT_f64_rr(RetVT, Op0, Op1);
11504 case MVT::v4f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_v4f16_rr(RetVT, Op0, Op1);
11505 case MVT::v8f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_v8f16_rr(RetVT, Op0, Op1);
11506 case MVT::v2f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_v2f32_rr(RetVT, Op0, Op1);
11507 case MVT::v4f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_v4f32_rr(RetVT, Op0, Op1);
11508 case MVT::v2f64: return fastEmit_ISD_FMINNUM_IEEE_MVT_v2f64_rr(RetVT, Op0, Op1);
11509 default: return Register();
11510 }
11511}
11512
11513// FastEmit functions for ISD::FMUL.
11514
11515Register fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11516 if (RetVT.SimpleTy != MVT::f16)
11517 return Register();
11518 if ((Subtarget->hasFullFP16())) {
11519 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11520 }
11521 return Register();
11522}
11523
11524Register fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11525 if (RetVT.SimpleTy != MVT::f32)
11526 return Register();
11527 if ((Subtarget->hasFPARMv8())) {
11528 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11529 }
11530 return Register();
11531}
11532
11533Register fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11534 if (RetVT.SimpleTy != MVT::f64)
11535 return Register();
11536 if ((Subtarget->hasFPARMv8())) {
11537 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11538 }
11539 return Register();
11540}
11541
11542Register fastEmit_ISD_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11543 if (RetVT.SimpleTy != MVT::v4f16)
11544 return Register();
11545 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11546 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11547 }
11548 return Register();
11549}
11550
11551Register fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11552 if (RetVT.SimpleTy != MVT::v8f16)
11553 return Register();
11554 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11555 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11556 }
11557 return Register();
11558}
11559
11560Register fastEmit_ISD_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11561 if (RetVT.SimpleTy != MVT::v2f32)
11562 return Register();
11563 if ((Subtarget->isNeonAvailable())) {
11564 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11565 }
11566 return Register();
11567}
11568
11569Register fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11570 if (RetVT.SimpleTy != MVT::v4f32)
11571 return Register();
11572 if ((Subtarget->isNeonAvailable())) {
11573 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11574 }
11575 return Register();
11576}
11577
11578Register fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11579 if (RetVT.SimpleTy != MVT::v2f64)
11580 return Register();
11581 if ((Subtarget->isNeonAvailable())) {
11582 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11583 }
11584 return Register();
11585}
11586
11587Register fastEmit_ISD_FMUL_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11588 if (RetVT.SimpleTy != MVT::nxv8f16)
11589 return Register();
11590 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11591 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11592 }
11593 return Register();
11594}
11595
11596Register fastEmit_ISD_FMUL_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11597 if (RetVT.SimpleTy != MVT::nxv8bf16)
11598 return Register();
11599 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
11600 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFMUL_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11601 }
11602 return Register();
11603}
11604
11605Register fastEmit_ISD_FMUL_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11606 if (RetVT.SimpleTy != MVT::nxv4f32)
11607 return Register();
11608 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11609 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11610 }
11611 return Register();
11612}
11613
11614Register fastEmit_ISD_FMUL_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11615 if (RetVT.SimpleTy != MVT::nxv2f64)
11616 return Register();
11617 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11618 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11619 }
11620 return Register();
11621}
11622
11623Register fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11624 switch (VT.SimpleTy) {
11625 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
11626 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
11627 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
11628 case MVT::v4f16: return fastEmit_ISD_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
11629 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
11630 case MVT::v2f32: return fastEmit_ISD_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
11631 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
11632 case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
11633 case MVT::nxv8f16: return fastEmit_ISD_FMUL_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11634 case MVT::nxv8bf16: return fastEmit_ISD_FMUL_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11635 case MVT::nxv4f32: return fastEmit_ISD_FMUL_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11636 case MVT::nxv2f64: return fastEmit_ISD_FMUL_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11637 default: return Register();
11638 }
11639}
11640
11641// FastEmit functions for ISD::FSUB.
11642
11643Register fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11644 if (RetVT.SimpleTy != MVT::f16)
11645 return Register();
11646 if ((Subtarget->hasFullFP16())) {
11647 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11648 }
11649 return Register();
11650}
11651
11652Register fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11653 if (RetVT.SimpleTy != MVT::f32)
11654 return Register();
11655 if ((Subtarget->hasFPARMv8())) {
11656 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11657 }
11658 return Register();
11659}
11660
11661Register fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11662 if (RetVT.SimpleTy != MVT::f64)
11663 return Register();
11664 if ((Subtarget->hasFPARMv8())) {
11665 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11666 }
11667 return Register();
11668}
11669
11670Register fastEmit_ISD_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11671 if (RetVT.SimpleTy != MVT::v4f16)
11672 return Register();
11673 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11674 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11675 }
11676 return Register();
11677}
11678
11679Register fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11680 if (RetVT.SimpleTy != MVT::v8f16)
11681 return Register();
11682 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11683 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11684 }
11685 return Register();
11686}
11687
11688Register fastEmit_ISD_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11689 if (RetVT.SimpleTy != MVT::v2f32)
11690 return Register();
11691 if ((Subtarget->isNeonAvailable())) {
11692 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11693 }
11694 return Register();
11695}
11696
11697Register fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11698 if (RetVT.SimpleTy != MVT::v4f32)
11699 return Register();
11700 if ((Subtarget->isNeonAvailable())) {
11701 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11702 }
11703 return Register();
11704}
11705
11706Register fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11707 if (RetVT.SimpleTy != MVT::v2f64)
11708 return Register();
11709 if ((Subtarget->isNeonAvailable())) {
11710 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11711 }
11712 return Register();
11713}
11714
11715Register fastEmit_ISD_FSUB_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11716 if (RetVT.SimpleTy != MVT::nxv8f16)
11717 return Register();
11718 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11719 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11720 }
11721 return Register();
11722}
11723
11724Register fastEmit_ISD_FSUB_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11725 if (RetVT.SimpleTy != MVT::nxv8bf16)
11726 return Register();
11727 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
11728 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFSUB_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11729 }
11730 return Register();
11731}
11732
11733Register fastEmit_ISD_FSUB_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11734 if (RetVT.SimpleTy != MVT::nxv4f32)
11735 return Register();
11736 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11737 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11738 }
11739 return Register();
11740}
11741
11742Register fastEmit_ISD_FSUB_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11743 if (RetVT.SimpleTy != MVT::nxv2f64)
11744 return Register();
11745 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11746 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11747 }
11748 return Register();
11749}
11750
11751Register fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11752 switch (VT.SimpleTy) {
11753 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
11754 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
11755 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
11756 case MVT::v4f16: return fastEmit_ISD_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
11757 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
11758 case MVT::v2f32: return fastEmit_ISD_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
11759 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
11760 case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
11761 case MVT::nxv8f16: return fastEmit_ISD_FSUB_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11762 case MVT::nxv8bf16: return fastEmit_ISD_FSUB_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11763 case MVT::nxv4f32: return fastEmit_ISD_FSUB_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11764 case MVT::nxv2f64: return fastEmit_ISD_FSUB_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11765 default: return Register();
11766 }
11767}
11768
11769// FastEmit functions for ISD::GET_ACTIVE_LANE_MASK.
11770
11771Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv2i1_rr(Register Op0, Register Op1) {
11772 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11773 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_D, RC: &AArch64::PPRRegClass, Op0, Op1);
11774 }
11775 return Register();
11776}
11777
11778Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv4i1_rr(Register Op0, Register Op1) {
11779 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11780 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_S, RC: &AArch64::PPRRegClass, Op0, Op1);
11781 }
11782 return Register();
11783}
11784
11785Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv8i1_rr(Register Op0, Register Op1) {
11786 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11787 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_H, RC: &AArch64::PPRRegClass, Op0, Op1);
11788 }
11789 return Register();
11790}
11791
11792Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv16i1_rr(Register Op0, Register Op1) {
11793 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11794 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_B, RC: &AArch64::PPRRegClass, Op0, Op1);
11795 }
11796 return Register();
11797}
11798
11799Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
11800switch (RetVT.SimpleTy) {
11801 case MVT::nxv2i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv2i1_rr(Op0, Op1);
11802 case MVT::nxv4i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv4i1_rr(Op0, Op1);
11803 case MVT::nxv8i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv8i1_rr(Op0, Op1);
11804 case MVT::nxv16i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv16i1_rr(Op0, Op1);
11805 default: return Register();
11806}
11807}
11808
11809Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv2i1_rr(Register Op0, Register Op1) {
11810 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11811 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_D, RC: &AArch64::PPRRegClass, Op0, Op1);
11812 }
11813 return Register();
11814}
11815
11816Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv4i1_rr(Register Op0, Register Op1) {
11817 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11818 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_S, RC: &AArch64::PPRRegClass, Op0, Op1);
11819 }
11820 return Register();
11821}
11822
11823Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv8i1_rr(Register Op0, Register Op1) {
11824 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11825 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_H, RC: &AArch64::PPRRegClass, Op0, Op1);
11826 }
11827 return Register();
11828}
11829
11830Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv16i1_rr(Register Op0, Register Op1) {
11831 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11832 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_B, RC: &AArch64::PPRRegClass, Op0, Op1);
11833 }
11834 return Register();
11835}
11836
11837Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11838switch (RetVT.SimpleTy) {
11839 case MVT::nxv2i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv2i1_rr(Op0, Op1);
11840 case MVT::nxv4i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv4i1_rr(Op0, Op1);
11841 case MVT::nxv8i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv8i1_rr(Op0, Op1);
11842 case MVT::nxv16i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv16i1_rr(Op0, Op1);
11843 default: return Register();
11844}
11845}
11846
11847Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11848 switch (VT.SimpleTy) {
11849 case MVT::i32: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_rr(RetVT, Op0, Op1);
11850 case MVT::i64: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_rr(RetVT, Op0, Op1);
11851 default: return Register();
11852 }
11853}
11854
11855// FastEmit functions for ISD::MUL.
11856
11857Register fastEmit_ISD_MUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
11858 if (RetVT.SimpleTy != MVT::v8i8)
11859 return Register();
11860 if ((Subtarget->isNeonAvailable())) {
11861 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
11862 }
11863 return Register();
11864}
11865
11866Register fastEmit_ISD_MUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
11867 if (RetVT.SimpleTy != MVT::v16i8)
11868 return Register();
11869 if ((Subtarget->isNeonAvailable())) {
11870 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
11871 }
11872 return Register();
11873}
11874
11875Register fastEmit_ISD_MUL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
11876 if (RetVT.SimpleTy != MVT::v4i16)
11877 return Register();
11878 if ((Subtarget->isNeonAvailable())) {
11879 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11880 }
11881 return Register();
11882}
11883
11884Register fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
11885 if (RetVT.SimpleTy != MVT::v8i16)
11886 return Register();
11887 if ((Subtarget->isNeonAvailable())) {
11888 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11889 }
11890 return Register();
11891}
11892
11893Register fastEmit_ISD_MUL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
11894 if (RetVT.SimpleTy != MVT::v2i32)
11895 return Register();
11896 if ((Subtarget->isNeonAvailable())) {
11897 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11898 }
11899 return Register();
11900}
11901
11902Register fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
11903 if (RetVT.SimpleTy != MVT::v4i32)
11904 return Register();
11905 if ((Subtarget->isNeonAvailable())) {
11906 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11907 }
11908 return Register();
11909}
11910
11911Register fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11912 switch (VT.SimpleTy) {
11913 case MVT::v8i8: return fastEmit_ISD_MUL_MVT_v8i8_rr(RetVT, Op0, Op1);
11914 case MVT::v16i8: return fastEmit_ISD_MUL_MVT_v16i8_rr(RetVT, Op0, Op1);
11915 case MVT::v4i16: return fastEmit_ISD_MUL_MVT_v4i16_rr(RetVT, Op0, Op1);
11916 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
11917 case MVT::v2i32: return fastEmit_ISD_MUL_MVT_v2i32_rr(RetVT, Op0, Op1);
11918 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
11919 default: return Register();
11920 }
11921}
11922
11923// FastEmit functions for ISD::MULHS.
11924
11925Register fastEmit_ISD_MULHS_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11926 if (RetVT.SimpleTy != MVT::i64)
11927 return Register();
11928 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULHrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
11929}
11930
11931Register fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11932 switch (VT.SimpleTy) {
11933 case MVT::i64: return fastEmit_ISD_MULHS_MVT_i64_rr(RetVT, Op0, Op1);
11934 default: return Register();
11935 }
11936}
11937
11938// FastEmit functions for ISD::MULHU.
11939
11940Register fastEmit_ISD_MULHU_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11941 if (RetVT.SimpleTy != MVT::i64)
11942 return Register();
11943 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULHrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
11944}
11945
11946Register fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11947 switch (VT.SimpleTy) {
11948 case MVT::i64: return fastEmit_ISD_MULHU_MVT_i64_rr(RetVT, Op0, Op1);
11949 default: return Register();
11950 }
11951}
11952
11953// FastEmit functions for ISD::OR.
11954
11955Register fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
11956 if (RetVT.SimpleTy != MVT::i32)
11957 return Register();
11958 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
11959}
11960
11961Register fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11962 if (RetVT.SimpleTy != MVT::i64)
11963 return Register();
11964 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
11965}
11966
11967Register fastEmit_ISD_OR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
11968 if (RetVT.SimpleTy != MVT::v8i8)
11969 return Register();
11970 if ((Subtarget->isNeonAvailable())) {
11971 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
11972 }
11973 return Register();
11974}
11975
11976Register fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
11977 if (RetVT.SimpleTy != MVT::v16i8)
11978 return Register();
11979 if ((Subtarget->isNeonAvailable())) {
11980 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
11981 }
11982 return Register();
11983}
11984
11985Register fastEmit_ISD_OR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
11986 if (RetVT.SimpleTy != MVT::v4i16)
11987 return Register();
11988 if ((Subtarget->isNeonAvailable())) {
11989 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
11990 }
11991 return Register();
11992}
11993
11994Register fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
11995 if (RetVT.SimpleTy != MVT::v8i16)
11996 return Register();
11997 if ((Subtarget->isNeonAvailable())) {
11998 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
11999 }
12000 return Register();
12001}
12002
12003Register fastEmit_ISD_OR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12004 if (RetVT.SimpleTy != MVT::v2i32)
12005 return Register();
12006 if ((Subtarget->isNeonAvailable())) {
12007 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12008 }
12009 return Register();
12010}
12011
12012Register fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12013 if (RetVT.SimpleTy != MVT::v4i32)
12014 return Register();
12015 if ((Subtarget->isNeonAvailable())) {
12016 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12017 }
12018 return Register();
12019}
12020
12021Register fastEmit_ISD_OR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12022 if (RetVT.SimpleTy != MVT::v1i64)
12023 return Register();
12024 if ((Subtarget->isNeonAvailable())) {
12025 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12026 }
12027 return Register();
12028}
12029
12030Register fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12031 if (RetVT.SimpleTy != MVT::v2i64)
12032 return Register();
12033 if ((Subtarget->isNeonAvailable())) {
12034 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12035 }
12036 return Register();
12037}
12038
12039Register fastEmit_ISD_OR_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12040 if (RetVT.SimpleTy != MVT::nxv16i8)
12041 return Register();
12042 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12043 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12044 }
12045 return Register();
12046}
12047
12048Register fastEmit_ISD_OR_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12049 if (RetVT.SimpleTy != MVT::nxv8i16)
12050 return Register();
12051 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12052 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12053 }
12054 return Register();
12055}
12056
12057Register fastEmit_ISD_OR_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12058 if (RetVT.SimpleTy != MVT::nxv4i32)
12059 return Register();
12060 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12061 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12062 }
12063 return Register();
12064}
12065
12066Register fastEmit_ISD_OR_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12067 if (RetVT.SimpleTy != MVT::nxv2i64)
12068 return Register();
12069 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12070 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12071 }
12072 return Register();
12073}
12074
12075Register fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12076 switch (VT.SimpleTy) {
12077 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
12078 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op1);
12079 case MVT::v8i8: return fastEmit_ISD_OR_MVT_v8i8_rr(RetVT, Op0, Op1);
12080 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
12081 case MVT::v4i16: return fastEmit_ISD_OR_MVT_v4i16_rr(RetVT, Op0, Op1);
12082 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
12083 case MVT::v2i32: return fastEmit_ISD_OR_MVT_v2i32_rr(RetVT, Op0, Op1);
12084 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
12085 case MVT::v1i64: return fastEmit_ISD_OR_MVT_v1i64_rr(RetVT, Op0, Op1);
12086 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
12087 case MVT::nxv16i8: return fastEmit_ISD_OR_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12088 case MVT::nxv8i16: return fastEmit_ISD_OR_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12089 case MVT::nxv4i32: return fastEmit_ISD_OR_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12090 case MVT::nxv2i64: return fastEmit_ISD_OR_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12091 default: return Register();
12092 }
12093}
12094
12095// FastEmit functions for ISD::ROTR.
12096
12097Register fastEmit_ISD_ROTR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12098 if (RetVT.SimpleTy != MVT::i64)
12099 return Register();
12100 return fastEmitInst_rr(MachineInstOpcode: AArch64::RORVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12101}
12102
12103Register fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12104 switch (VT.SimpleTy) {
12105 case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_rr(RetVT, Op0, Op1);
12106 default: return Register();
12107 }
12108}
12109
12110// FastEmit functions for ISD::SADDSAT.
12111
12112Register fastEmit_ISD_SADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12113 if (RetVT.SimpleTy != MVT::v8i8)
12114 return Register();
12115 if ((Subtarget->isNeonAvailable())) {
12116 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12117 }
12118 return Register();
12119}
12120
12121Register fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12122 if (RetVT.SimpleTy != MVT::v16i8)
12123 return Register();
12124 if ((Subtarget->isNeonAvailable())) {
12125 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12126 }
12127 return Register();
12128}
12129
12130Register fastEmit_ISD_SADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12131 if (RetVT.SimpleTy != MVT::v4i16)
12132 return Register();
12133 if ((Subtarget->isNeonAvailable())) {
12134 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12135 }
12136 return Register();
12137}
12138
12139Register fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12140 if (RetVT.SimpleTy != MVT::v8i16)
12141 return Register();
12142 if ((Subtarget->isNeonAvailable())) {
12143 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12144 }
12145 return Register();
12146}
12147
12148Register fastEmit_ISD_SADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12149 if (RetVT.SimpleTy != MVT::v2i32)
12150 return Register();
12151 if ((Subtarget->isNeonAvailable())) {
12152 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12153 }
12154 return Register();
12155}
12156
12157Register fastEmit_ISD_SADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12158 if (RetVT.SimpleTy != MVT::v4i32)
12159 return Register();
12160 if ((Subtarget->isNeonAvailable())) {
12161 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12162 }
12163 return Register();
12164}
12165
12166Register fastEmit_ISD_SADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12167 if (RetVT.SimpleTy != MVT::v1i64)
12168 return Register();
12169 if ((Subtarget->isNeonAvailable())) {
12170 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
12171 }
12172 return Register();
12173}
12174
12175Register fastEmit_ISD_SADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12176 if (RetVT.SimpleTy != MVT::v2i64)
12177 return Register();
12178 if ((Subtarget->isNeonAvailable())) {
12179 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12180 }
12181 return Register();
12182}
12183
12184Register fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12185 if (RetVT.SimpleTy != MVT::nxv16i8)
12186 return Register();
12187 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12188 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
12189 }
12190 return Register();
12191}
12192
12193Register fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12194 if (RetVT.SimpleTy != MVT::nxv8i16)
12195 return Register();
12196 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12197 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
12198 }
12199 return Register();
12200}
12201
12202Register fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12203 if (RetVT.SimpleTy != MVT::nxv4i32)
12204 return Register();
12205 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12206 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
12207 }
12208 return Register();
12209}
12210
12211Register fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12212 if (RetVT.SimpleTy != MVT::nxv2i64)
12213 return Register();
12214 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12215 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
12216 }
12217 return Register();
12218}
12219
12220Register fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12221 switch (VT.SimpleTy) {
12222 case MVT::v8i8: return fastEmit_ISD_SADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
12223 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12224 case MVT::v4i16: return fastEmit_ISD_SADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
12225 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12226 case MVT::v2i32: return fastEmit_ISD_SADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
12227 case MVT::v4i32: return fastEmit_ISD_SADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
12228 case MVT::v1i64: return fastEmit_ISD_SADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
12229 case MVT::v2i64: return fastEmit_ISD_SADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
12230 case MVT::nxv16i8: return fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12231 case MVT::nxv8i16: return fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12232 case MVT::nxv4i32: return fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12233 case MVT::nxv2i64: return fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12234 default: return Register();
12235 }
12236}
12237
12238// FastEmit functions for ISD::SDIV.
12239
12240Register fastEmit_ISD_SDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12241 if (RetVT.SimpleTy != MVT::i32)
12242 return Register();
12243 return fastEmitInst_rr(MachineInstOpcode: AArch64::SDIVWr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12244}
12245
12246Register fastEmit_ISD_SDIV_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12247 if (RetVT.SimpleTy != MVT::i64)
12248 return Register();
12249 return fastEmitInst_rr(MachineInstOpcode: AArch64::SDIVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12250}
12251
12252Register fastEmit_ISD_SDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12253 switch (VT.SimpleTy) {
12254 case MVT::i32: return fastEmit_ISD_SDIV_MVT_i32_rr(RetVT, Op0, Op1);
12255 case MVT::i64: return fastEmit_ISD_SDIV_MVT_i64_rr(RetVT, Op0, Op1);
12256 default: return Register();
12257 }
12258}
12259
12260// FastEmit functions for ISD::SHL.
12261
12262Register fastEmit_ISD_SHL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12263 if (RetVT.SimpleTy != MVT::i64)
12264 return Register();
12265 return fastEmitInst_rr(MachineInstOpcode: AArch64::LSLVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12266}
12267
12268Register fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12269 switch (VT.SimpleTy) {
12270 case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_rr(RetVT, Op0, Op1);
12271 default: return Register();
12272 }
12273}
12274
12275// FastEmit functions for ISD::SMAX.
12276
12277Register fastEmit_ISD_SMAX_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12278 if (RetVT.SimpleTy != MVT::i32)
12279 return Register();
12280 if ((Subtarget->hasCSSC())) {
12281 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12282 }
12283 return Register();
12284}
12285
12286Register fastEmit_ISD_SMAX_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12287 if (RetVT.SimpleTy != MVT::i64)
12288 return Register();
12289 if ((Subtarget->hasCSSC())) {
12290 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12291 }
12292 return Register();
12293}
12294
12295Register fastEmit_ISD_SMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12296 if (RetVT.SimpleTy != MVT::v8i8)
12297 return Register();
12298 if ((Subtarget->isNeonAvailable())) {
12299 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12300 }
12301 return Register();
12302}
12303
12304Register fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12305 if (RetVT.SimpleTy != MVT::v16i8)
12306 return Register();
12307 if ((Subtarget->isNeonAvailable())) {
12308 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12309 }
12310 return Register();
12311}
12312
12313Register fastEmit_ISD_SMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12314 if (RetVT.SimpleTy != MVT::v4i16)
12315 return Register();
12316 if ((Subtarget->isNeonAvailable())) {
12317 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12318 }
12319 return Register();
12320}
12321
12322Register fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12323 if (RetVT.SimpleTy != MVT::v8i16)
12324 return Register();
12325 if ((Subtarget->isNeonAvailable())) {
12326 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12327 }
12328 return Register();
12329}
12330
12331Register fastEmit_ISD_SMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12332 if (RetVT.SimpleTy != MVT::v2i32)
12333 return Register();
12334 if ((Subtarget->isNeonAvailable())) {
12335 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12336 }
12337 return Register();
12338}
12339
12340Register fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12341 if (RetVT.SimpleTy != MVT::v4i32)
12342 return Register();
12343 if ((Subtarget->isNeonAvailable())) {
12344 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12345 }
12346 return Register();
12347}
12348
12349Register fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12350 switch (VT.SimpleTy) {
12351 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_rr(RetVT, Op0, Op1);
12352 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_rr(RetVT, Op0, Op1);
12353 case MVT::v8i8: return fastEmit_ISD_SMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
12354 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
12355 case MVT::v4i16: return fastEmit_ISD_SMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
12356 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
12357 case MVT::v2i32: return fastEmit_ISD_SMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
12358 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
12359 default: return Register();
12360 }
12361}
12362
12363// FastEmit functions for ISD::SMIN.
12364
12365Register fastEmit_ISD_SMIN_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12366 if (RetVT.SimpleTy != MVT::i32)
12367 return Register();
12368 if ((Subtarget->hasCSSC())) {
12369 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12370 }
12371 return Register();
12372}
12373
12374Register fastEmit_ISD_SMIN_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12375 if (RetVT.SimpleTy != MVT::i64)
12376 return Register();
12377 if ((Subtarget->hasCSSC())) {
12378 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12379 }
12380 return Register();
12381}
12382
12383Register fastEmit_ISD_SMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12384 if (RetVT.SimpleTy != MVT::v8i8)
12385 return Register();
12386 if ((Subtarget->isNeonAvailable())) {
12387 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12388 }
12389 return Register();
12390}
12391
12392Register fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12393 if (RetVT.SimpleTy != MVT::v16i8)
12394 return Register();
12395 if ((Subtarget->isNeonAvailable())) {
12396 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12397 }
12398 return Register();
12399}
12400
12401Register fastEmit_ISD_SMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12402 if (RetVT.SimpleTy != MVT::v4i16)
12403 return Register();
12404 if ((Subtarget->isNeonAvailable())) {
12405 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12406 }
12407 return Register();
12408}
12409
12410Register fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12411 if (RetVT.SimpleTy != MVT::v8i16)
12412 return Register();
12413 if ((Subtarget->isNeonAvailable())) {
12414 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12415 }
12416 return Register();
12417}
12418
12419Register fastEmit_ISD_SMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12420 if (RetVT.SimpleTy != MVT::v2i32)
12421 return Register();
12422 if ((Subtarget->isNeonAvailable())) {
12423 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12424 }
12425 return Register();
12426}
12427
12428Register fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12429 if (RetVT.SimpleTy != MVT::v4i32)
12430 return Register();
12431 if ((Subtarget->isNeonAvailable())) {
12432 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12433 }
12434 return Register();
12435}
12436
12437Register fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12438 switch (VT.SimpleTy) {
12439 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_rr(RetVT, Op0, Op1);
12440 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_rr(RetVT, Op0, Op1);
12441 case MVT::v8i8: return fastEmit_ISD_SMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
12442 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
12443 case MVT::v4i16: return fastEmit_ISD_SMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
12444 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
12445 case MVT::v2i32: return fastEmit_ISD_SMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
12446 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
12447 default: return Register();
12448 }
12449}
12450
12451// FastEmit functions for ISD::SRA.
12452
12453Register fastEmit_ISD_SRA_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12454 if (RetVT.SimpleTy != MVT::i64)
12455 return Register();
12456 return fastEmitInst_rr(MachineInstOpcode: AArch64::ASRVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12457}
12458
12459Register fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12460 switch (VT.SimpleTy) {
12461 case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_rr(RetVT, Op0, Op1);
12462 default: return Register();
12463 }
12464}
12465
12466// FastEmit functions for ISD::SRL.
12467
12468Register fastEmit_ISD_SRL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12469 if (RetVT.SimpleTy != MVT::i64)
12470 return Register();
12471 return fastEmitInst_rr(MachineInstOpcode: AArch64::LSRVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12472}
12473
12474Register fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12475 switch (VT.SimpleTy) {
12476 case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_rr(RetVT, Op0, Op1);
12477 default: return Register();
12478 }
12479}
12480
12481// FastEmit functions for ISD::SSUBSAT.
12482
12483Register fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12484 if (RetVT.SimpleTy != MVT::v8i8)
12485 return Register();
12486 if ((Subtarget->isNeonAvailable())) {
12487 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12488 }
12489 return Register();
12490}
12491
12492Register fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12493 if (RetVT.SimpleTy != MVT::v16i8)
12494 return Register();
12495 if ((Subtarget->isNeonAvailable())) {
12496 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12497 }
12498 return Register();
12499}
12500
12501Register fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12502 if (RetVT.SimpleTy != MVT::v4i16)
12503 return Register();
12504 if ((Subtarget->isNeonAvailable())) {
12505 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12506 }
12507 return Register();
12508}
12509
12510Register fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12511 if (RetVT.SimpleTy != MVT::v8i16)
12512 return Register();
12513 if ((Subtarget->isNeonAvailable())) {
12514 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12515 }
12516 return Register();
12517}
12518
12519Register fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12520 if (RetVT.SimpleTy != MVT::v2i32)
12521 return Register();
12522 if ((Subtarget->isNeonAvailable())) {
12523 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12524 }
12525 return Register();
12526}
12527
12528Register fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12529 if (RetVT.SimpleTy != MVT::v4i32)
12530 return Register();
12531 if ((Subtarget->isNeonAvailable())) {
12532 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12533 }
12534 return Register();
12535}
12536
12537Register fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12538 if (RetVT.SimpleTy != MVT::v1i64)
12539 return Register();
12540 if ((Subtarget->isNeonAvailable())) {
12541 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
12542 }
12543 return Register();
12544}
12545
12546Register fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12547 if (RetVT.SimpleTy != MVT::v2i64)
12548 return Register();
12549 if ((Subtarget->isNeonAvailable())) {
12550 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12551 }
12552 return Register();
12553}
12554
12555Register fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12556 if (RetVT.SimpleTy != MVT::nxv16i8)
12557 return Register();
12558 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12559 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
12560 }
12561 return Register();
12562}
12563
12564Register fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12565 if (RetVT.SimpleTy != MVT::nxv8i16)
12566 return Register();
12567 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12568 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
12569 }
12570 return Register();
12571}
12572
12573Register fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12574 if (RetVT.SimpleTy != MVT::nxv4i32)
12575 return Register();
12576 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12577 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
12578 }
12579 return Register();
12580}
12581
12582Register fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12583 if (RetVT.SimpleTy != MVT::nxv2i64)
12584 return Register();
12585 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12586 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
12587 }
12588 return Register();
12589}
12590
12591Register fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12592 switch (VT.SimpleTy) {
12593 case MVT::v8i8: return fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
12594 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12595 case MVT::v4i16: return fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
12596 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12597 case MVT::v2i32: return fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
12598 case MVT::v4i32: return fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
12599 case MVT::v1i64: return fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
12600 case MVT::v2i64: return fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
12601 case MVT::nxv16i8: return fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12602 case MVT::nxv8i16: return fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12603 case MVT::nxv4i32: return fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12604 case MVT::nxv2i64: return fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12605 default: return Register();
12606 }
12607}
12608
12609// FastEmit functions for ISD::STRICT_FADD.
12610
12611Register fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12612 if (RetVT.SimpleTy != MVT::f16)
12613 return Register();
12614 if ((Subtarget->hasFullFP16())) {
12615 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12616 }
12617 return Register();
12618}
12619
12620Register fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12621 if (RetVT.SimpleTy != MVT::f32)
12622 return Register();
12623 if ((Subtarget->hasFPARMv8())) {
12624 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12625 }
12626 return Register();
12627}
12628
12629Register fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12630 if (RetVT.SimpleTy != MVT::f64)
12631 return Register();
12632 if ((Subtarget->hasFPARMv8())) {
12633 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12634 }
12635 return Register();
12636}
12637
12638Register fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12639 if (RetVT.SimpleTy != MVT::v4f16)
12640 return Register();
12641 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12642 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12643 }
12644 return Register();
12645}
12646
12647Register fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12648 if (RetVT.SimpleTy != MVT::v8f16)
12649 return Register();
12650 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12651 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12652 }
12653 return Register();
12654}
12655
12656Register fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12657 if (RetVT.SimpleTy != MVT::v2f32)
12658 return Register();
12659 if ((Subtarget->isNeonAvailable())) {
12660 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12661 }
12662 return Register();
12663}
12664
12665Register fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12666 if (RetVT.SimpleTy != MVT::v4f32)
12667 return Register();
12668 if ((Subtarget->isNeonAvailable())) {
12669 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12670 }
12671 return Register();
12672}
12673
12674Register fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12675 if (RetVT.SimpleTy != MVT::v2f64)
12676 return Register();
12677 if ((Subtarget->isNeonAvailable())) {
12678 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12679 }
12680 return Register();
12681}
12682
12683Register fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12684 switch (VT.SimpleTy) {
12685 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1);
12686 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1);
12687 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1);
12688 case MVT::v4f16: return fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
12689 case MVT::v8f16: return fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
12690 case MVT::v2f32: return fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
12691 case MVT::v4f32: return fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
12692 case MVT::v2f64: return fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
12693 default: return Register();
12694 }
12695}
12696
12697// FastEmit functions for ISD::STRICT_FDIV.
12698
12699Register fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12700 if (RetVT.SimpleTy != MVT::f16)
12701 return Register();
12702 if ((Subtarget->hasFullFP16())) {
12703 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12704 }
12705 return Register();
12706}
12707
12708Register fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12709 if (RetVT.SimpleTy != MVT::f32)
12710 return Register();
12711 if ((Subtarget->hasFPARMv8())) {
12712 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12713 }
12714 return Register();
12715}
12716
12717Register fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12718 if (RetVT.SimpleTy != MVT::f64)
12719 return Register();
12720 if ((Subtarget->hasFPARMv8())) {
12721 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12722 }
12723 return Register();
12724}
12725
12726Register fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12727 if (RetVT.SimpleTy != MVT::v4f16)
12728 return Register();
12729 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12730 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12731 }
12732 return Register();
12733}
12734
12735Register fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12736 if (RetVT.SimpleTy != MVT::v8f16)
12737 return Register();
12738 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12739 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12740 }
12741 return Register();
12742}
12743
12744Register fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12745 if (RetVT.SimpleTy != MVT::v2f32)
12746 return Register();
12747 if ((Subtarget->isNeonAvailable())) {
12748 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12749 }
12750 return Register();
12751}
12752
12753Register fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12754 if (RetVT.SimpleTy != MVT::v4f32)
12755 return Register();
12756 if ((Subtarget->isNeonAvailable())) {
12757 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12758 }
12759 return Register();
12760}
12761
12762Register fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12763 if (RetVT.SimpleTy != MVT::v2f64)
12764 return Register();
12765 if ((Subtarget->isNeonAvailable())) {
12766 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12767 }
12768 return Register();
12769}
12770
12771Register fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12772 switch (VT.SimpleTy) {
12773 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
12774 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
12775 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
12776 case MVT::v4f16: return fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1);
12777 case MVT::v8f16: return fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
12778 case MVT::v2f32: return fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1);
12779 case MVT::v4f32: return fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
12780 case MVT::v2f64: return fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
12781 default: return Register();
12782 }
12783}
12784
12785// FastEmit functions for ISD::STRICT_FMAXIMUM.
12786
12787Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12788 if (RetVT.SimpleTy != MVT::f16)
12789 return Register();
12790 if ((Subtarget->hasFullFP16())) {
12791 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12792 }
12793 return Register();
12794}
12795
12796Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12797 if (RetVT.SimpleTy != MVT::f32)
12798 return Register();
12799 if ((Subtarget->hasFPARMv8())) {
12800 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12801 }
12802 return Register();
12803}
12804
12805Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12806 if (RetVT.SimpleTy != MVT::f64)
12807 return Register();
12808 if ((Subtarget->hasFPARMv8())) {
12809 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12810 }
12811 return Register();
12812}
12813
12814Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12815 if (RetVT.SimpleTy != MVT::v4f16)
12816 return Register();
12817 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12818 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12819 }
12820 return Register();
12821}
12822
12823Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12824 if (RetVT.SimpleTy != MVT::v8f16)
12825 return Register();
12826 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12827 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12828 }
12829 return Register();
12830}
12831
12832Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12833 if (RetVT.SimpleTy != MVT::v2f32)
12834 return Register();
12835 if ((Subtarget->isNeonAvailable())) {
12836 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12837 }
12838 return Register();
12839}
12840
12841Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12842 if (RetVT.SimpleTy != MVT::v4f32)
12843 return Register();
12844 if ((Subtarget->isNeonAvailable())) {
12845 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12846 }
12847 return Register();
12848}
12849
12850Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12851 if (RetVT.SimpleTy != MVT::v2f64)
12852 return Register();
12853 if ((Subtarget->isNeonAvailable())) {
12854 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12855 }
12856 return Register();
12857}
12858
12859Register fastEmit_ISD_STRICT_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12860 switch (VT.SimpleTy) {
12861 case MVT::f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1);
12862 case MVT::f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1);
12863 case MVT::f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1);
12864 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
12865 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
12866 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
12867 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
12868 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
12869 default: return Register();
12870 }
12871}
12872
12873// FastEmit functions for ISD::STRICT_FMAXNUM.
12874
12875Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12876 if (RetVT.SimpleTy != MVT::f16)
12877 return Register();
12878 if ((Subtarget->hasFullFP16())) {
12879 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12880 }
12881 return Register();
12882}
12883
12884Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12885 if (RetVT.SimpleTy != MVT::f32)
12886 return Register();
12887 if ((Subtarget->hasFPARMv8())) {
12888 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12889 }
12890 return Register();
12891}
12892
12893Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12894 if (RetVT.SimpleTy != MVT::f64)
12895 return Register();
12896 if ((Subtarget->hasFPARMv8())) {
12897 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12898 }
12899 return Register();
12900}
12901
12902Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12903 if (RetVT.SimpleTy != MVT::v4f16)
12904 return Register();
12905 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12906 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12907 }
12908 return Register();
12909}
12910
12911Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12912 if (RetVT.SimpleTy != MVT::v8f16)
12913 return Register();
12914 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12915 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12916 }
12917 return Register();
12918}
12919
12920Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12921 if (RetVT.SimpleTy != MVT::v2f32)
12922 return Register();
12923 if ((Subtarget->isNeonAvailable())) {
12924 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12925 }
12926 return Register();
12927}
12928
12929Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12930 if (RetVT.SimpleTy != MVT::v4f32)
12931 return Register();
12932 if ((Subtarget->isNeonAvailable())) {
12933 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12934 }
12935 return Register();
12936}
12937
12938Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12939 if (RetVT.SimpleTy != MVT::v2f64)
12940 return Register();
12941 if ((Subtarget->isNeonAvailable())) {
12942 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12943 }
12944 return Register();
12945}
12946
12947Register fastEmit_ISD_STRICT_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12948 switch (VT.SimpleTy) {
12949 case MVT::f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
12950 case MVT::f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
12951 case MVT::f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
12952 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
12953 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
12954 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
12955 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
12956 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
12957 default: return Register();
12958 }
12959}
12960
12961// FastEmit functions for ISD::STRICT_FMINIMUM.
12962
12963Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12964 if (RetVT.SimpleTy != MVT::f16)
12965 return Register();
12966 if ((Subtarget->hasFullFP16())) {
12967 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12968 }
12969 return Register();
12970}
12971
12972Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12973 if (RetVT.SimpleTy != MVT::f32)
12974 return Register();
12975 if ((Subtarget->hasFPARMv8())) {
12976 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12977 }
12978 return Register();
12979}
12980
12981Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12982 if (RetVT.SimpleTy != MVT::f64)
12983 return Register();
12984 if ((Subtarget->hasFPARMv8())) {
12985 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12986 }
12987 return Register();
12988}
12989
12990Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12991 if (RetVT.SimpleTy != MVT::v4f16)
12992 return Register();
12993 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12994 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12995 }
12996 return Register();
12997}
12998
12999Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13000 if (RetVT.SimpleTy != MVT::v8f16)
13001 return Register();
13002 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13003 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13004 }
13005 return Register();
13006}
13007
13008Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13009 if (RetVT.SimpleTy != MVT::v2f32)
13010 return Register();
13011 if ((Subtarget->isNeonAvailable())) {
13012 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13013 }
13014 return Register();
13015}
13016
13017Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13018 if (RetVT.SimpleTy != MVT::v4f32)
13019 return Register();
13020 if ((Subtarget->isNeonAvailable())) {
13021 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13022 }
13023 return Register();
13024}
13025
13026Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13027 if (RetVT.SimpleTy != MVT::v2f64)
13028 return Register();
13029 if ((Subtarget->isNeonAvailable())) {
13030 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13031 }
13032 return Register();
13033}
13034
13035Register fastEmit_ISD_STRICT_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13036 switch (VT.SimpleTy) {
13037 case MVT::f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1);
13038 case MVT::f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1);
13039 case MVT::f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1);
13040 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13041 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13042 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13043 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13044 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13045 default: return Register();
13046 }
13047}
13048
13049// FastEmit functions for ISD::STRICT_FMINNUM.
13050
13051Register fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13052 if (RetVT.SimpleTy != MVT::f16)
13053 return Register();
13054 if ((Subtarget->hasFullFP16())) {
13055 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13056 }
13057 return Register();
13058}
13059
13060Register fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13061 if (RetVT.SimpleTy != MVT::f32)
13062 return Register();
13063 if ((Subtarget->hasFPARMv8())) {
13064 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13065 }
13066 return Register();
13067}
13068
13069Register fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13070 if (RetVT.SimpleTy != MVT::f64)
13071 return Register();
13072 if ((Subtarget->hasFPARMv8())) {
13073 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13074 }
13075 return Register();
13076}
13077
13078Register fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13079 if (RetVT.SimpleTy != MVT::v4f16)
13080 return Register();
13081 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13082 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13083 }
13084 return Register();
13085}
13086
13087Register fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13088 if (RetVT.SimpleTy != MVT::v8f16)
13089 return Register();
13090 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13091 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13092 }
13093 return Register();
13094}
13095
13096Register fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13097 if (RetVT.SimpleTy != MVT::v2f32)
13098 return Register();
13099 if ((Subtarget->isNeonAvailable())) {
13100 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13101 }
13102 return Register();
13103}
13104
13105Register fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13106 if (RetVT.SimpleTy != MVT::v4f32)
13107 return Register();
13108 if ((Subtarget->isNeonAvailable())) {
13109 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13110 }
13111 return Register();
13112}
13113
13114Register fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13115 if (RetVT.SimpleTy != MVT::v2f64)
13116 return Register();
13117 if ((Subtarget->isNeonAvailable())) {
13118 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13119 }
13120 return Register();
13121}
13122
13123Register fastEmit_ISD_STRICT_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13124 switch (VT.SimpleTy) {
13125 case MVT::f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
13126 case MVT::f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
13127 case MVT::f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
13128 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13129 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13130 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13131 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13132 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13133 default: return Register();
13134 }
13135}
13136
13137// FastEmit functions for ISD::STRICT_FMUL.
13138
13139Register fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13140 if (RetVT.SimpleTy != MVT::f16)
13141 return Register();
13142 if ((Subtarget->hasFullFP16())) {
13143 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13144 }
13145 return Register();
13146}
13147
13148Register fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13149 if (RetVT.SimpleTy != MVT::f32)
13150 return Register();
13151 if ((Subtarget->hasFPARMv8())) {
13152 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13153 }
13154 return Register();
13155}
13156
13157Register fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13158 if (RetVT.SimpleTy != MVT::f64)
13159 return Register();
13160 if ((Subtarget->hasFPARMv8())) {
13161 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13162 }
13163 return Register();
13164}
13165
13166Register fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13167 if (RetVT.SimpleTy != MVT::v4f16)
13168 return Register();
13169 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13170 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13171 }
13172 return Register();
13173}
13174
13175Register fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13176 if (RetVT.SimpleTy != MVT::v8f16)
13177 return Register();
13178 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13179 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13180 }
13181 return Register();
13182}
13183
13184Register fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13185 if (RetVT.SimpleTy != MVT::v2f32)
13186 return Register();
13187 if ((Subtarget->isNeonAvailable())) {
13188 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13189 }
13190 return Register();
13191}
13192
13193Register fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13194 if (RetVT.SimpleTy != MVT::v4f32)
13195 return Register();
13196 if ((Subtarget->isNeonAvailable())) {
13197 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13198 }
13199 return Register();
13200}
13201
13202Register fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13203 if (RetVT.SimpleTy != MVT::v2f64)
13204 return Register();
13205 if ((Subtarget->isNeonAvailable())) {
13206 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13207 }
13208 return Register();
13209}
13210
13211Register fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13212 switch (VT.SimpleTy) {
13213 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
13214 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
13215 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
13216 case MVT::v4f16: return fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
13217 case MVT::v8f16: return fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
13218 case MVT::v2f32: return fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
13219 case MVT::v4f32: return fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
13220 case MVT::v2f64: return fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
13221 default: return Register();
13222 }
13223}
13224
13225// FastEmit functions for ISD::STRICT_FSUB.
13226
13227Register fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13228 if (RetVT.SimpleTy != MVT::f16)
13229 return Register();
13230 if ((Subtarget->hasFullFP16())) {
13231 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13232 }
13233 return Register();
13234}
13235
13236Register fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13237 if (RetVT.SimpleTy != MVT::f32)
13238 return Register();
13239 if ((Subtarget->hasFPARMv8())) {
13240 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13241 }
13242 return Register();
13243}
13244
13245Register fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13246 if (RetVT.SimpleTy != MVT::f64)
13247 return Register();
13248 if ((Subtarget->hasFPARMv8())) {
13249 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13250 }
13251 return Register();
13252}
13253
13254Register fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13255 if (RetVT.SimpleTy != MVT::v4f16)
13256 return Register();
13257 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13258 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13259 }
13260 return Register();
13261}
13262
13263Register fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13264 if (RetVT.SimpleTy != MVT::v8f16)
13265 return Register();
13266 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13267 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13268 }
13269 return Register();
13270}
13271
13272Register fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13273 if (RetVT.SimpleTy != MVT::v2f32)
13274 return Register();
13275 if ((Subtarget->isNeonAvailable())) {
13276 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13277 }
13278 return Register();
13279}
13280
13281Register fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13282 if (RetVT.SimpleTy != MVT::v4f32)
13283 return Register();
13284 if ((Subtarget->isNeonAvailable())) {
13285 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13286 }
13287 return Register();
13288}
13289
13290Register fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13291 if (RetVT.SimpleTy != MVT::v2f64)
13292 return Register();
13293 if ((Subtarget->isNeonAvailable())) {
13294 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13295 }
13296 return Register();
13297}
13298
13299Register fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13300 switch (VT.SimpleTy) {
13301 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
13302 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
13303 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
13304 case MVT::v4f16: return fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
13305 case MVT::v8f16: return fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
13306 case MVT::v2f32: return fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
13307 case MVT::v4f32: return fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
13308 case MVT::v2f64: return fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
13309 default: return Register();
13310 }
13311}
13312
13313// FastEmit functions for ISD::SUB.
13314
13315Register fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13316 if (RetVT.SimpleTy != MVT::i32)
13317 return Register();
13318 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBSWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13319}
13320
13321Register fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13322 if (RetVT.SimpleTy != MVT::i64)
13323 return Register();
13324 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBSXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13325}
13326
13327Register fastEmit_ISD_SUB_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13328 if (RetVT.SimpleTy != MVT::v8i8)
13329 return Register();
13330 if ((Subtarget->isNeonAvailable())) {
13331 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13332 }
13333 return Register();
13334}
13335
13336Register fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13337 if (RetVT.SimpleTy != MVT::v16i8)
13338 return Register();
13339 if ((Subtarget->isNeonAvailable())) {
13340 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13341 }
13342 return Register();
13343}
13344
13345Register fastEmit_ISD_SUB_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13346 if (RetVT.SimpleTy != MVT::v4i16)
13347 return Register();
13348 if ((Subtarget->isNeonAvailable())) {
13349 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13350 }
13351 return Register();
13352}
13353
13354Register fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13355 if (RetVT.SimpleTy != MVT::v8i16)
13356 return Register();
13357 if ((Subtarget->isNeonAvailable())) {
13358 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13359 }
13360 return Register();
13361}
13362
13363Register fastEmit_ISD_SUB_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13364 if (RetVT.SimpleTy != MVT::v2i32)
13365 return Register();
13366 if ((Subtarget->isNeonAvailable())) {
13367 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13368 }
13369 return Register();
13370}
13371
13372Register fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13373 if (RetVT.SimpleTy != MVT::v4i32)
13374 return Register();
13375 if ((Subtarget->isNeonAvailable())) {
13376 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13377 }
13378 return Register();
13379}
13380
13381Register fastEmit_ISD_SUB_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13382 if (RetVT.SimpleTy != MVT::v1i64)
13383 return Register();
13384 if ((Subtarget->isNeonAvailable())) {
13385 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13386 }
13387 return Register();
13388}
13389
13390Register fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13391 if (RetVT.SimpleTy != MVT::v2i64)
13392 return Register();
13393 if ((Subtarget->isNeonAvailable())) {
13394 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13395 }
13396 return Register();
13397}
13398
13399Register fastEmit_ISD_SUB_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13400 if (RetVT.SimpleTy != MVT::nxv16i8)
13401 return Register();
13402 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13403 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13404 }
13405 return Register();
13406}
13407
13408Register fastEmit_ISD_SUB_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13409 if (RetVT.SimpleTy != MVT::nxv8i16)
13410 return Register();
13411 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13412 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13413 }
13414 return Register();
13415}
13416
13417Register fastEmit_ISD_SUB_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13418 if (RetVT.SimpleTy != MVT::nxv4i32)
13419 return Register();
13420 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13421 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13422 }
13423 return Register();
13424}
13425
13426Register fastEmit_ISD_SUB_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13427 if (RetVT.SimpleTy != MVT::nxv2i64)
13428 return Register();
13429 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13430 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13431 }
13432 return Register();
13433}
13434
13435Register fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13436 switch (VT.SimpleTy) {
13437 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
13438 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op1);
13439 case MVT::v8i8: return fastEmit_ISD_SUB_MVT_v8i8_rr(RetVT, Op0, Op1);
13440 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
13441 case MVT::v4i16: return fastEmit_ISD_SUB_MVT_v4i16_rr(RetVT, Op0, Op1);
13442 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
13443 case MVT::v2i32: return fastEmit_ISD_SUB_MVT_v2i32_rr(RetVT, Op0, Op1);
13444 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
13445 case MVT::v1i64: return fastEmit_ISD_SUB_MVT_v1i64_rr(RetVT, Op0, Op1);
13446 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
13447 case MVT::nxv16i8: return fastEmit_ISD_SUB_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13448 case MVT::nxv8i16: return fastEmit_ISD_SUB_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13449 case MVT::nxv4i32: return fastEmit_ISD_SUB_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13450 case MVT::nxv2i64: return fastEmit_ISD_SUB_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13451 default: return Register();
13452 }
13453}
13454
13455// FastEmit functions for ISD::UADDSAT.
13456
13457Register fastEmit_ISD_UADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13458 if (RetVT.SimpleTy != MVT::v8i8)
13459 return Register();
13460 if ((Subtarget->isNeonAvailable())) {
13461 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13462 }
13463 return Register();
13464}
13465
13466Register fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13467 if (RetVT.SimpleTy != MVT::v16i8)
13468 return Register();
13469 if ((Subtarget->isNeonAvailable())) {
13470 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13471 }
13472 return Register();
13473}
13474
13475Register fastEmit_ISD_UADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13476 if (RetVT.SimpleTy != MVT::v4i16)
13477 return Register();
13478 if ((Subtarget->isNeonAvailable())) {
13479 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13480 }
13481 return Register();
13482}
13483
13484Register fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13485 if (RetVT.SimpleTy != MVT::v8i16)
13486 return Register();
13487 if ((Subtarget->isNeonAvailable())) {
13488 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13489 }
13490 return Register();
13491}
13492
13493Register fastEmit_ISD_UADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13494 if (RetVT.SimpleTy != MVT::v2i32)
13495 return Register();
13496 if ((Subtarget->isNeonAvailable())) {
13497 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13498 }
13499 return Register();
13500}
13501
13502Register fastEmit_ISD_UADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13503 if (RetVT.SimpleTy != MVT::v4i32)
13504 return Register();
13505 if ((Subtarget->isNeonAvailable())) {
13506 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13507 }
13508 return Register();
13509}
13510
13511Register fastEmit_ISD_UADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13512 if (RetVT.SimpleTy != MVT::v1i64)
13513 return Register();
13514 if ((Subtarget->isNeonAvailable())) {
13515 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13516 }
13517 return Register();
13518}
13519
13520Register fastEmit_ISD_UADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13521 if (RetVT.SimpleTy != MVT::v2i64)
13522 return Register();
13523 if ((Subtarget->isNeonAvailable())) {
13524 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13525 }
13526 return Register();
13527}
13528
13529Register fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13530 if (RetVT.SimpleTy != MVT::nxv16i8)
13531 return Register();
13532 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13533 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13534 }
13535 return Register();
13536}
13537
13538Register fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13539 if (RetVT.SimpleTy != MVT::nxv8i16)
13540 return Register();
13541 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13542 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13543 }
13544 return Register();
13545}
13546
13547Register fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13548 if (RetVT.SimpleTy != MVT::nxv4i32)
13549 return Register();
13550 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13551 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13552 }
13553 return Register();
13554}
13555
13556Register fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13557 if (RetVT.SimpleTy != MVT::nxv2i64)
13558 return Register();
13559 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13560 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13561 }
13562 return Register();
13563}
13564
13565Register fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13566 switch (VT.SimpleTy) {
13567 case MVT::v8i8: return fastEmit_ISD_UADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
13568 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13569 case MVT::v4i16: return fastEmit_ISD_UADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
13570 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13571 case MVT::v2i32: return fastEmit_ISD_UADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
13572 case MVT::v4i32: return fastEmit_ISD_UADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
13573 case MVT::v1i64: return fastEmit_ISD_UADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
13574 case MVT::v2i64: return fastEmit_ISD_UADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
13575 case MVT::nxv16i8: return fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13576 case MVT::nxv8i16: return fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13577 case MVT::nxv4i32: return fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13578 case MVT::nxv2i64: return fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13579 default: return Register();
13580 }
13581}
13582
13583// FastEmit functions for ISD::UDIV.
13584
13585Register fastEmit_ISD_UDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13586 if (RetVT.SimpleTy != MVT::i32)
13587 return Register();
13588 return fastEmitInst_rr(MachineInstOpcode: AArch64::UDIVWr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13589}
13590
13591Register fastEmit_ISD_UDIV_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13592 if (RetVT.SimpleTy != MVT::i64)
13593 return Register();
13594 return fastEmitInst_rr(MachineInstOpcode: AArch64::UDIVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13595}
13596
13597Register fastEmit_ISD_UDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13598 switch (VT.SimpleTy) {
13599 case MVT::i32: return fastEmit_ISD_UDIV_MVT_i32_rr(RetVT, Op0, Op1);
13600 case MVT::i64: return fastEmit_ISD_UDIV_MVT_i64_rr(RetVT, Op0, Op1);
13601 default: return Register();
13602 }
13603}
13604
13605// FastEmit functions for ISD::UMAX.
13606
13607Register fastEmit_ISD_UMAX_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13608 if (RetVT.SimpleTy != MVT::i32)
13609 return Register();
13610 if ((Subtarget->hasCSSC())) {
13611 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13612 }
13613 return Register();
13614}
13615
13616Register fastEmit_ISD_UMAX_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13617 if (RetVT.SimpleTy != MVT::i64)
13618 return Register();
13619 if ((Subtarget->hasCSSC())) {
13620 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13621 }
13622 return Register();
13623}
13624
13625Register fastEmit_ISD_UMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13626 if (RetVT.SimpleTy != MVT::v8i8)
13627 return Register();
13628 if ((Subtarget->isNeonAvailable())) {
13629 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13630 }
13631 return Register();
13632}
13633
13634Register fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13635 if (RetVT.SimpleTy != MVT::v16i8)
13636 return Register();
13637 if ((Subtarget->isNeonAvailable())) {
13638 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13639 }
13640 return Register();
13641}
13642
13643Register fastEmit_ISD_UMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13644 if (RetVT.SimpleTy != MVT::v4i16)
13645 return Register();
13646 if ((Subtarget->isNeonAvailable())) {
13647 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13648 }
13649 return Register();
13650}
13651
13652Register fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13653 if (RetVT.SimpleTy != MVT::v8i16)
13654 return Register();
13655 if ((Subtarget->isNeonAvailable())) {
13656 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13657 }
13658 return Register();
13659}
13660
13661Register fastEmit_ISD_UMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13662 if (RetVT.SimpleTy != MVT::v2i32)
13663 return Register();
13664 if ((Subtarget->isNeonAvailable())) {
13665 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13666 }
13667 return Register();
13668}
13669
13670Register fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13671 if (RetVT.SimpleTy != MVT::v4i32)
13672 return Register();
13673 if ((Subtarget->isNeonAvailable())) {
13674 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13675 }
13676 return Register();
13677}
13678
13679Register fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13680 switch (VT.SimpleTy) {
13681 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_rr(RetVT, Op0, Op1);
13682 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_rr(RetVT, Op0, Op1);
13683 case MVT::v8i8: return fastEmit_ISD_UMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
13684 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
13685 case MVT::v4i16: return fastEmit_ISD_UMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
13686 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
13687 case MVT::v2i32: return fastEmit_ISD_UMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
13688 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
13689 default: return Register();
13690 }
13691}
13692
13693// FastEmit functions for ISD::UMIN.
13694
13695Register fastEmit_ISD_UMIN_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13696 if (RetVT.SimpleTy != MVT::i32)
13697 return Register();
13698 if ((Subtarget->hasCSSC())) {
13699 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13700 }
13701 return Register();
13702}
13703
13704Register fastEmit_ISD_UMIN_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13705 if (RetVT.SimpleTy != MVT::i64)
13706 return Register();
13707 if ((Subtarget->hasCSSC())) {
13708 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13709 }
13710 return Register();
13711}
13712
13713Register fastEmit_ISD_UMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13714 if (RetVT.SimpleTy != MVT::v8i8)
13715 return Register();
13716 if ((Subtarget->isNeonAvailable())) {
13717 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13718 }
13719 return Register();
13720}
13721
13722Register fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13723 if (RetVT.SimpleTy != MVT::v16i8)
13724 return Register();
13725 if ((Subtarget->isNeonAvailable())) {
13726 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13727 }
13728 return Register();
13729}
13730
13731Register fastEmit_ISD_UMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13732 if (RetVT.SimpleTy != MVT::v4i16)
13733 return Register();
13734 if ((Subtarget->isNeonAvailable())) {
13735 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13736 }
13737 return Register();
13738}
13739
13740Register fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13741 if (RetVT.SimpleTy != MVT::v8i16)
13742 return Register();
13743 if ((Subtarget->isNeonAvailable())) {
13744 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13745 }
13746 return Register();
13747}
13748
13749Register fastEmit_ISD_UMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13750 if (RetVT.SimpleTy != MVT::v2i32)
13751 return Register();
13752 if ((Subtarget->isNeonAvailable())) {
13753 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13754 }
13755 return Register();
13756}
13757
13758Register fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13759 if (RetVT.SimpleTy != MVT::v4i32)
13760 return Register();
13761 if ((Subtarget->isNeonAvailable())) {
13762 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13763 }
13764 return Register();
13765}
13766
13767Register fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13768 switch (VT.SimpleTy) {
13769 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_rr(RetVT, Op0, Op1);
13770 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_rr(RetVT, Op0, Op1);
13771 case MVT::v8i8: return fastEmit_ISD_UMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
13772 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
13773 case MVT::v4i16: return fastEmit_ISD_UMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
13774 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
13775 case MVT::v2i32: return fastEmit_ISD_UMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
13776 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
13777 default: return Register();
13778 }
13779}
13780
13781// FastEmit functions for ISD::USUBSAT.
13782
13783Register fastEmit_ISD_USUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13784 if (RetVT.SimpleTy != MVT::v8i8)
13785 return Register();
13786 if ((Subtarget->isNeonAvailable())) {
13787 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13788 }
13789 return Register();
13790}
13791
13792Register fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13793 if (RetVT.SimpleTy != MVT::v16i8)
13794 return Register();
13795 if ((Subtarget->isNeonAvailable())) {
13796 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13797 }
13798 return Register();
13799}
13800
13801Register fastEmit_ISD_USUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13802 if (RetVT.SimpleTy != MVT::v4i16)
13803 return Register();
13804 if ((Subtarget->isNeonAvailable())) {
13805 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13806 }
13807 return Register();
13808}
13809
13810Register fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13811 if (RetVT.SimpleTy != MVT::v8i16)
13812 return Register();
13813 if ((Subtarget->isNeonAvailable())) {
13814 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13815 }
13816 return Register();
13817}
13818
13819Register fastEmit_ISD_USUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13820 if (RetVT.SimpleTy != MVT::v2i32)
13821 return Register();
13822 if ((Subtarget->isNeonAvailable())) {
13823 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13824 }
13825 return Register();
13826}
13827
13828Register fastEmit_ISD_USUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13829 if (RetVT.SimpleTy != MVT::v4i32)
13830 return Register();
13831 if ((Subtarget->isNeonAvailable())) {
13832 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13833 }
13834 return Register();
13835}
13836
13837Register fastEmit_ISD_USUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13838 if (RetVT.SimpleTy != MVT::v1i64)
13839 return Register();
13840 if ((Subtarget->isNeonAvailable())) {
13841 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13842 }
13843 return Register();
13844}
13845
13846Register fastEmit_ISD_USUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13847 if (RetVT.SimpleTy != MVT::v2i64)
13848 return Register();
13849 if ((Subtarget->isNeonAvailable())) {
13850 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13851 }
13852 return Register();
13853}
13854
13855Register fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13856 if (RetVT.SimpleTy != MVT::nxv16i8)
13857 return Register();
13858 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13859 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13860 }
13861 return Register();
13862}
13863
13864Register fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13865 if (RetVT.SimpleTy != MVT::nxv8i16)
13866 return Register();
13867 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13868 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13869 }
13870 return Register();
13871}
13872
13873Register fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13874 if (RetVT.SimpleTy != MVT::nxv4i32)
13875 return Register();
13876 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13877 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13878 }
13879 return Register();
13880}
13881
13882Register fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13883 if (RetVT.SimpleTy != MVT::nxv2i64)
13884 return Register();
13885 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13886 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13887 }
13888 return Register();
13889}
13890
13891Register fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13892 switch (VT.SimpleTy) {
13893 case MVT::v8i8: return fastEmit_ISD_USUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
13894 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13895 case MVT::v4i16: return fastEmit_ISD_USUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
13896 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13897 case MVT::v2i32: return fastEmit_ISD_USUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
13898 case MVT::v4i32: return fastEmit_ISD_USUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
13899 case MVT::v1i64: return fastEmit_ISD_USUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
13900 case MVT::v2i64: return fastEmit_ISD_USUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
13901 case MVT::nxv16i8: return fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13902 case MVT::nxv8i16: return fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13903 case MVT::nxv4i32: return fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13904 case MVT::nxv2i64: return fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13905 default: return Register();
13906 }
13907}
13908
13909// FastEmit functions for ISD::XOR.
13910
13911Register fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13912 if (RetVT.SimpleTy != MVT::i32)
13913 return Register();
13914 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13915}
13916
13917Register fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13918 if (RetVT.SimpleTy != MVT::i64)
13919 return Register();
13920 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13921}
13922
13923Register fastEmit_ISD_XOR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13924 if (RetVT.SimpleTy != MVT::v8i8)
13925 return Register();
13926 if ((Subtarget->isNeonAvailable())) {
13927 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13928 }
13929 return Register();
13930}
13931
13932Register fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13933 if (RetVT.SimpleTy != MVT::v16i8)
13934 return Register();
13935 if ((Subtarget->isNeonAvailable())) {
13936 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13937 }
13938 return Register();
13939}
13940
13941Register fastEmit_ISD_XOR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13942 if (RetVT.SimpleTy != MVT::v4i16)
13943 return Register();
13944 if ((Subtarget->isNeonAvailable())) {
13945 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13946 }
13947 return Register();
13948}
13949
13950Register fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13951 if (RetVT.SimpleTy != MVT::v8i16)
13952 return Register();
13953 if ((Subtarget->isNeonAvailable())) {
13954 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13955 }
13956 return Register();
13957}
13958
13959Register fastEmit_ISD_XOR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13960 if (RetVT.SimpleTy != MVT::v2i32)
13961 return Register();
13962 if ((Subtarget->isNeonAvailable())) {
13963 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13964 }
13965 return Register();
13966}
13967
13968Register fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13969 if (RetVT.SimpleTy != MVT::v4i32)
13970 return Register();
13971 if ((Subtarget->isNeonAvailable())) {
13972 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13973 }
13974 return Register();
13975}
13976
13977Register fastEmit_ISD_XOR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13978 if (RetVT.SimpleTy != MVT::v1i64)
13979 return Register();
13980 if ((Subtarget->isNeonAvailable())) {
13981 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13982 }
13983 return Register();
13984}
13985
13986Register fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13987 if (RetVT.SimpleTy != MVT::v2i64)
13988 return Register();
13989 if ((Subtarget->isNeonAvailable())) {
13990 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13991 }
13992 return Register();
13993}
13994
13995Register fastEmit_ISD_XOR_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13996 if (RetVT.SimpleTy != MVT::nxv16i8)
13997 return Register();
13998 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13999 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14000 }
14001 return Register();
14002}
14003
14004Register fastEmit_ISD_XOR_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14005 if (RetVT.SimpleTy != MVT::nxv8i16)
14006 return Register();
14007 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14008 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14009 }
14010 return Register();
14011}
14012
14013Register fastEmit_ISD_XOR_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14014 if (RetVT.SimpleTy != MVT::nxv4i32)
14015 return Register();
14016 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14017 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14018 }
14019 return Register();
14020}
14021
14022Register fastEmit_ISD_XOR_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14023 if (RetVT.SimpleTy != MVT::nxv2i64)
14024 return Register();
14025 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14026 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14027 }
14028 return Register();
14029}
14030
14031Register fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14032 switch (VT.SimpleTy) {
14033 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
14034 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op1);
14035 case MVT::v8i8: return fastEmit_ISD_XOR_MVT_v8i8_rr(RetVT, Op0, Op1);
14036 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
14037 case MVT::v4i16: return fastEmit_ISD_XOR_MVT_v4i16_rr(RetVT, Op0, Op1);
14038 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
14039 case MVT::v2i32: return fastEmit_ISD_XOR_MVT_v2i32_rr(RetVT, Op0, Op1);
14040 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
14041 case MVT::v1i64: return fastEmit_ISD_XOR_MVT_v1i64_rr(RetVT, Op0, Op1);
14042 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
14043 case MVT::nxv16i8: return fastEmit_ISD_XOR_MVT_nxv16i8_rr(RetVT, Op0, Op1);
14044 case MVT::nxv8i16: return fastEmit_ISD_XOR_MVT_nxv8i16_rr(RetVT, Op0, Op1);
14045 case MVT::nxv4i32: return fastEmit_ISD_XOR_MVT_nxv4i32_rr(RetVT, Op0, Op1);
14046 case MVT::nxv2i64: return fastEmit_ISD_XOR_MVT_nxv2i64_rr(RetVT, Op0, Op1);
14047 default: return Register();
14048 }
14049}
14050
14051// Top-level FastEmit function.
14052
14053Register fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, Register Op1) override {
14054 switch (Opcode) {
14055 case AArch64ISD::ADDHN: return fastEmit_AArch64ISD_ADDHN_rr(VT, RetVT, Op0, Op1);
14056 case AArch64ISD::ADDP: return fastEmit_AArch64ISD_ADDP_rr(VT, RetVT, Op0, Op1);
14057 case AArch64ISD::BIC: return fastEmit_AArch64ISD_BIC_rr(VT, RetVT, Op0, Op1);
14058 case AArch64ISD::CMTST: return fastEmit_AArch64ISD_CMTST_rr(VT, RetVT, Op0, Op1);
14059 case AArch64ISD::CTTZ_ELTS_ZERO_POISON: return fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_rr(VT, RetVT, Op0, Op1);
14060 case AArch64ISD::FCMEQ: return fastEmit_AArch64ISD_FCMEQ_rr(VT, RetVT, Op0, Op1);
14061 case AArch64ISD::FCMGE: return fastEmit_AArch64ISD_FCMGE_rr(VT, RetVT, Op0, Op1);
14062 case AArch64ISD::FCMGT: return fastEmit_AArch64ISD_FCMGT_rr(VT, RetVT, Op0, Op1);
14063 case AArch64ISD::FRECPS: return fastEmit_AArch64ISD_FRECPS_rr(VT, RetVT, Op0, Op1);
14064 case AArch64ISD::FRSQRTS: return fastEmit_AArch64ISD_FRSQRTS_rr(VT, RetVT, Op0, Op1);
14065 case AArch64ISD::PMULL: return fastEmit_AArch64ISD_PMULL_rr(VT, RetVT, Op0, Op1);
14066 case AArch64ISD::SMULL: return fastEmit_AArch64ISD_SMULL_rr(VT, RetVT, Op0, Op1);
14067 case AArch64ISD::SQADD: return fastEmit_AArch64ISD_SQADD_rr(VT, RetVT, Op0, Op1);
14068 case AArch64ISD::SQDMULH: return fastEmit_AArch64ISD_SQDMULH_rr(VT, RetVT, Op0, Op1);
14069 case AArch64ISD::SQDMULL: return fastEmit_AArch64ISD_SQDMULL_rr(VT, RetVT, Op0, Op1);
14070 case AArch64ISD::SQRDMULH: return fastEmit_AArch64ISD_SQRDMULH_rr(VT, RetVT, Op0, Op1);
14071 case AArch64ISD::SQRSHL: return fastEmit_AArch64ISD_SQRSHL_rr(VT, RetVT, Op0, Op1);
14072 case AArch64ISD::SQSHL: return fastEmit_AArch64ISD_SQSHL_rr(VT, RetVT, Op0, Op1);
14073 case AArch64ISD::SQSUB: return fastEmit_AArch64ISD_SQSUB_rr(VT, RetVT, Op0, Op1);
14074 case AArch64ISD::SUQADD: return fastEmit_AArch64ISD_SUQADD_rr(VT, RetVT, Op0, Op1);
14075 case AArch64ISD::TBL: return fastEmit_AArch64ISD_TBL_rr(VT, RetVT, Op0, Op1);
14076 case AArch64ISD::TRN1: return fastEmit_AArch64ISD_TRN1_rr(VT, RetVT, Op0, Op1);
14077 case AArch64ISD::TRN2: return fastEmit_AArch64ISD_TRN2_rr(VT, RetVT, Op0, Op1);
14078 case AArch64ISD::UMULL: return fastEmit_AArch64ISD_UMULL_rr(VT, RetVT, Op0, Op1);
14079 case AArch64ISD::UQADD: return fastEmit_AArch64ISD_UQADD_rr(VT, RetVT, Op0, Op1);
14080 case AArch64ISD::UQRSHL: return fastEmit_AArch64ISD_UQRSHL_rr(VT, RetVT, Op0, Op1);
14081 case AArch64ISD::UQSHL: return fastEmit_AArch64ISD_UQSHL_rr(VT, RetVT, Op0, Op1);
14082 case AArch64ISD::UQSUB: return fastEmit_AArch64ISD_UQSUB_rr(VT, RetVT, Op0, Op1);
14083 case AArch64ISD::USQADD: return fastEmit_AArch64ISD_USQADD_rr(VT, RetVT, Op0, Op1);
14084 case AArch64ISD::UZP1: return fastEmit_AArch64ISD_UZP1_rr(VT, RetVT, Op0, Op1);
14085 case AArch64ISD::UZP2: return fastEmit_AArch64ISD_UZP2_rr(VT, RetVT, Op0, Op1);
14086 case AArch64ISD::ZIP1: return fastEmit_AArch64ISD_ZIP1_rr(VT, RetVT, Op0, Op1);
14087 case AArch64ISD::ZIP2: return fastEmit_AArch64ISD_ZIP2_rr(VT, RetVT, Op0, Op1);
14088 case ISD::ABDS: return fastEmit_ISD_ABDS_rr(VT, RetVT, Op0, Op1);
14089 case ISD::ABDU: return fastEmit_ISD_ABDU_rr(VT, RetVT, Op0, Op1);
14090 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
14091 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
14092 case ISD::AVGCEILS: return fastEmit_ISD_AVGCEILS_rr(VT, RetVT, Op0, Op1);
14093 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
14094 case ISD::AVGFLOORS: return fastEmit_ISD_AVGFLOORS_rr(VT, RetVT, Op0, Op1);
14095 case ISD::AVGFLOORU: return fastEmit_ISD_AVGFLOORU_rr(VT, RetVT, Op0, Op1);
14096 case ISD::CLMUL: return fastEmit_ISD_CLMUL_rr(VT, RetVT, Op0, Op1);
14097 case ISD::CONCAT_VECTORS: return fastEmit_ISD_CONCAT_VECTORS_rr(VT, RetVT, Op0, Op1);
14098 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
14099 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
14100 case ISD::FMAXIMUM: return fastEmit_ISD_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
14101 case ISD::FMAXNUM: return fastEmit_ISD_FMAXNUM_rr(VT, RetVT, Op0, Op1);
14102 case ISD::FMAXNUM_IEEE: return fastEmit_ISD_FMAXNUM_IEEE_rr(VT, RetVT, Op0, Op1);
14103 case ISD::FMINIMUM: return fastEmit_ISD_FMINIMUM_rr(VT, RetVT, Op0, Op1);
14104 case ISD::FMINNUM: return fastEmit_ISD_FMINNUM_rr(VT, RetVT, Op0, Op1);
14105 case ISD::FMINNUM_IEEE: return fastEmit_ISD_FMINNUM_IEEE_rr(VT, RetVT, Op0, Op1);
14106 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
14107 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
14108 case ISD::GET_ACTIVE_LANE_MASK: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_rr(VT, RetVT, Op0, Op1);
14109 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
14110 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
14111 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
14112 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
14113 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
14114 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
14115 case ISD::SDIV: return fastEmit_ISD_SDIV_rr(VT, RetVT, Op0, Op1);
14116 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
14117 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
14118 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
14119 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
14120 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
14121 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
14122 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1);
14123 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1);
14124 case ISD::STRICT_FMAXIMUM: return fastEmit_ISD_STRICT_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
14125 case ISD::STRICT_FMAXNUM: return fastEmit_ISD_STRICT_FMAXNUM_rr(VT, RetVT, Op0, Op1);
14126 case ISD::STRICT_FMINIMUM: return fastEmit_ISD_STRICT_FMINIMUM_rr(VT, RetVT, Op0, Op1);
14127 case ISD::STRICT_FMINNUM: return fastEmit_ISD_STRICT_FMINNUM_rr(VT, RetVT, Op0, Op1);
14128 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1);
14129 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1);
14130 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
14131 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
14132 case ISD::UDIV: return fastEmit_ISD_UDIV_rr(VT, RetVT, Op0, Op1);
14133 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
14134 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
14135 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
14136 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
14137 default: return Register();
14138 }
14139}
14140
14141// FastEmit functions for AArch64ISD::DUPLANE64.
14142
14143Register fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14144 if (RetVT.SimpleTy != MVT::v2i64)
14145 return Register();
14146 if ((Subtarget->isNeonAvailable())) {
14147 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i64lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14148 }
14149 return Register();
14150}
14151
14152Register fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14153 if (RetVT.SimpleTy != MVT::v2f64)
14154 return Register();
14155 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i64lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14156}
14157
14158Register fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14159 switch (VT.SimpleTy) {
14160 case MVT::v2i64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14161 case MVT::v2f64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14162 default: return Register();
14163 }
14164}
14165
14166// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14167
14168Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14169 if (RetVT.SimpleTy != MVT::i64)
14170 return Register();
14171 if ((Subtarget->isNeonAvailable())) {
14172 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi64, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14173 }
14174 return Register();
14175}
14176
14177Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14178 if (RetVT.SimpleTy != MVT::f64)
14179 return Register();
14180 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi64, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14181}
14182
14183Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14184 switch (VT.SimpleTy) {
14185 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14186 case MVT::v2f64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14187 default: return Register();
14188 }
14189}
14190
14191// Top-level FastEmit function.
14192
14193Register fastEmit_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14194 switch (Opcode) {
14195 case AArch64ISD::DUPLANE64: return fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1);
14196 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1);
14197 default: return Register();
14198 }
14199}
14200
14201// FastEmit functions for AArch64ISD::DUPLANE32.
14202
14203Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14204 if ((Subtarget->isNeonAvailable())) {
14205 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i32lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14206 }
14207 return Register();
14208}
14209
14210Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14211 if ((Subtarget->isNeonAvailable())) {
14212 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i32lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14213 }
14214 return Register();
14215}
14216
14217Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14218switch (RetVT.SimpleTy) {
14219 case MVT::v2i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Op0, imm1);
14220 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Op0, imm1);
14221 default: return Register();
14222}
14223}
14224
14225Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14226 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i32lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14227}
14228
14229Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14230 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i32lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14231}
14232
14233Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14234switch (RetVT.SimpleTy) {
14235 case MVT::v2f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Op0, imm1);
14236 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Op0, imm1);
14237 default: return Register();
14238}
14239}
14240
14241Register fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14242 switch (VT.SimpleTy) {
14243 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14244 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14245 default: return Register();
14246 }
14247}
14248
14249// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14250
14251Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14252 if (RetVT.SimpleTy != MVT::i32)
14253 return Register();
14254 if ((Subtarget->isNeonAvailable())) {
14255 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi32, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14256 }
14257 return Register();
14258}
14259
14260Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14261 if (RetVT.SimpleTy != MVT::f32)
14262 return Register();
14263 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi32, RC: &AArch64::FPR32RegClass, Op0, Imm: imm1);
14264}
14265
14266Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14267 switch (VT.SimpleTy) {
14268 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14269 case MVT::v4f32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14270 default: return Register();
14271 }
14272}
14273
14274// Top-level FastEmit function.
14275
14276Register fastEmit_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14277 switch (Opcode) {
14278 case AArch64ISD::DUPLANE32: return fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1);
14279 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1);
14280 default: return Register();
14281 }
14282}
14283
14284// FastEmit functions for AArch64ISD::DUPLANE16.
14285
14286Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14287 if ((Subtarget->isNeonAvailable())) {
14288 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14289 }
14290 return Register();
14291}
14292
14293Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14294 if ((Subtarget->isNeonAvailable())) {
14295 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14296 }
14297 return Register();
14298}
14299
14300Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14301switch (RetVT.SimpleTy) {
14302 case MVT::v4i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Op0, imm1);
14303 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Op0, imm1);
14304 default: return Register();
14305}
14306}
14307
14308Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14309 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14310}
14311
14312Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14313 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14314}
14315
14316Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14317switch (RetVT.SimpleTy) {
14318 case MVT::v4f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Op0, imm1);
14319 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Op0, imm1);
14320 default: return Register();
14321}
14322}
14323
14324Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14325 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14326}
14327
14328Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14329 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14330}
14331
14332Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14333switch (RetVT.SimpleTy) {
14334 case MVT::v4bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(Op0, imm1);
14335 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(Op0, imm1);
14336 default: return Register();
14337}
14338}
14339
14340Register fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14341 switch (VT.SimpleTy) {
14342 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14343 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14344 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14345 default: return Register();
14346 }
14347}
14348
14349// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14350
14351Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14352 if (RetVT.SimpleTy != MVT::i32)
14353 return Register();
14354 if ((Subtarget->isNeonAvailable())) {
14355 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi16, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14356 }
14357 return Register();
14358}
14359
14360Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14361 if (RetVT.SimpleTy != MVT::f16)
14362 return Register();
14363 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi16, RC: &AArch64::FPR16RegClass, Op0, Imm: imm1);
14364}
14365
14366Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14367 if (RetVT.SimpleTy != MVT::bf16)
14368 return Register();
14369 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi16, RC: &AArch64::FPR16RegClass, Op0, Imm: imm1);
14370}
14371
14372Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14373 switch (VT.SimpleTy) {
14374 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14375 case MVT::v8f16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14376 case MVT::v8bf16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14377 default: return Register();
14378 }
14379}
14380
14381// Top-level FastEmit function.
14382
14383Register fastEmit_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14384 switch (Opcode) {
14385 case AArch64ISD::DUPLANE16: return fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1);
14386 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1);
14387 default: return Register();
14388 }
14389}
14390
14391// FastEmit functions for AArch64ISD::DUPLANE8.
14392
14393Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Register Op0, uint64_t imm1) {
14394 if ((Subtarget->isNeonAvailable())) {
14395 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i8lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14396 }
14397 return Register();
14398}
14399
14400Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Register Op0, uint64_t imm1) {
14401 if ((Subtarget->isNeonAvailable())) {
14402 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv16i8lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14403 }
14404 return Register();
14405}
14406
14407Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, Register Op0, uint64_t imm1) {
14408switch (RetVT.SimpleTy) {
14409 case MVT::v8i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Op0, imm1);
14410 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Op0, imm1);
14411 default: return Register();
14412}
14413}
14414
14415Register fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14416 switch (VT.SimpleTy) {
14417 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1);
14418 default: return Register();
14419 }
14420}
14421
14422// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14423
14424Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, Register Op0, uint64_t imm1) {
14425 if (RetVT.SimpleTy != MVT::i32)
14426 return Register();
14427 if ((Subtarget->isNeonAvailable())) {
14428 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi8, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14429 }
14430 return Register();
14431}
14432
14433Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14434 switch (VT.SimpleTy) {
14435 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1);
14436 default: return Register();
14437 }
14438}
14439
14440// Top-level FastEmit function.
14441
14442Register fastEmit_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14443 switch (Opcode) {
14444 case AArch64ISD::DUPLANE8: return fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1);
14445 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1);
14446 default: return Register();
14447 }
14448}
14449
14450// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14451
14452Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14453 if (RetVT.SimpleTy != MVT::i32)
14454 return Register();
14455 if ((Subtarget->hasNEON())) {
14456 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi8_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14457 }
14458 return Register();
14459}
14460
14461Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14462 if (RetVT.SimpleTy != MVT::i32)
14463 return Register();
14464 if ((Subtarget->hasNEON())) {
14465 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi16_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14466 }
14467 return Register();
14468}
14469
14470Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14471 if (RetVT.SimpleTy != MVT::i32)
14472 return Register();
14473 if ((Subtarget->hasNEON())) {
14474 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi32_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14475 }
14476 return Register();
14477}
14478
14479Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14480 if (RetVT.SimpleTy != MVT::i64)
14481 return Register();
14482 if ((Subtarget->hasNEON())) {
14483 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi64_idx0, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14484 }
14485 return Register();
14486}
14487
14488Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14489 switch (VT.SimpleTy) {
14490 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14491 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14492 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14493 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14494 default: return Register();
14495 }
14496}
14497
14498// Top-level FastEmit function.
14499
14500Register fastEmit_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14501 switch (Opcode) {
14502 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(VT, RetVT, Op0, imm1);
14503 default: return Register();
14504 }
14505}
14506
14507// FastEmit functions for ISD::SMAX.
14508
14509Register fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14510 if (RetVT.SimpleTy != MVT::i32)
14511 return Register();
14512 if ((Subtarget->hasCSSC())) {
14513 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMAXWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14514 }
14515 return Register();
14516}
14517
14518Register fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14519 switch (VT.SimpleTy) {
14520 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1);
14521 default: return Register();
14522 }
14523}
14524
14525// FastEmit functions for ISD::SMIN.
14526
14527Register fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14528 if (RetVT.SimpleTy != MVT::i32)
14529 return Register();
14530 if ((Subtarget->hasCSSC())) {
14531 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMINWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14532 }
14533 return Register();
14534}
14535
14536Register fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14537 switch (VT.SimpleTy) {
14538 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1);
14539 default: return Register();
14540 }
14541}
14542
14543// Top-level FastEmit function.
14544
14545Register fastEmit_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14546 switch (Opcode) {
14547 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1);
14548 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1);
14549 default: return Register();
14550 }
14551}
14552
14553// FastEmit functions for ISD::SMAX.
14554
14555Register fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14556 if (RetVT.SimpleTy != MVT::i64)
14557 return Register();
14558 if ((Subtarget->hasCSSC())) {
14559 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMAXXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14560 }
14561 return Register();
14562}
14563
14564Register fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14565 switch (VT.SimpleTy) {
14566 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1);
14567 default: return Register();
14568 }
14569}
14570
14571// FastEmit functions for ISD::SMIN.
14572
14573Register fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14574 if (RetVT.SimpleTy != MVT::i64)
14575 return Register();
14576 if ((Subtarget->hasCSSC())) {
14577 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMINXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14578 }
14579 return Register();
14580}
14581
14582Register fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14583 switch (VT.SimpleTy) {
14584 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1);
14585 default: return Register();
14586 }
14587}
14588
14589// Top-level FastEmit function.
14590
14591Register fastEmit_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14592 switch (Opcode) {
14593 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1);
14594 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1);
14595 default: return Register();
14596 }
14597}
14598
14599// FastEmit functions for ISD::UMAX.
14600
14601Register fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14602 if (RetVT.SimpleTy != MVT::i32)
14603 return Register();
14604 if ((Subtarget->hasCSSC())) {
14605 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMAXWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14606 }
14607 return Register();
14608}
14609
14610Register fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14611 switch (VT.SimpleTy) {
14612 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1);
14613 default: return Register();
14614 }
14615}
14616
14617// FastEmit functions for ISD::UMIN.
14618
14619Register fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14620 if (RetVT.SimpleTy != MVT::i32)
14621 return Register();
14622 if ((Subtarget->hasCSSC())) {
14623 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMINWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14624 }
14625 return Register();
14626}
14627
14628Register fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14629 switch (VT.SimpleTy) {
14630 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1);
14631 default: return Register();
14632 }
14633}
14634
14635// Top-level FastEmit function.
14636
14637Register fastEmit_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14638 switch (Opcode) {
14639 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1);
14640 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1);
14641 default: return Register();
14642 }
14643}
14644
14645// FastEmit functions for ISD::UMAX.
14646
14647Register fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14648 if (RetVT.SimpleTy != MVT::i64)
14649 return Register();
14650 if ((Subtarget->hasCSSC())) {
14651 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMAXXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14652 }
14653 return Register();
14654}
14655
14656Register fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14657 switch (VT.SimpleTy) {
14658 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1);
14659 default: return Register();
14660 }
14661}
14662
14663// FastEmit functions for ISD::UMIN.
14664
14665Register fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14666 if (RetVT.SimpleTy != MVT::i64)
14667 return Register();
14668 if ((Subtarget->hasCSSC())) {
14669 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMINXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14670 }
14671 return Register();
14672}
14673
14674Register fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14675 switch (VT.SimpleTy) {
14676 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1);
14677 default: return Register();
14678 }
14679}
14680
14681// Top-level FastEmit function.
14682
14683Register fastEmit_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14684 switch (Opcode) {
14685 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1);
14686 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1);
14687 default: return Register();
14688 }
14689}
14690
14691// FastEmit functions for ISD::Constant.
14692
14693Register fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
14694 if (RetVT.SimpleTy != MVT::i32)
14695 return Register();
14696 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVi32imm, RC: &AArch64::GPR32RegClass, Imm: imm0);
14697}
14698
14699Register fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) {
14700 if (RetVT.SimpleTy != MVT::i64)
14701 return Register();
14702 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVi64imm, RC: &AArch64::GPR64RegClass, Imm: imm0);
14703}
14704
14705Register fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
14706 switch (VT.SimpleTy) {
14707 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
14708 case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0);
14709 default: return Register();
14710 }
14711}
14712
14713// Top-level FastEmit function.
14714
14715Register fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
14716 if (VT == MVT::i32 && Predicate_imm0_255(Imm: imm0))
14717 if (Register Reg = fastEmit_i_Predicate_imm0_255(VT, RetVT, Opcode, imm0))
14718 return Reg;
14719
14720 if (VT == MVT::i32 && Predicate_simm6_32b(Imm: imm0))
14721 if (Register Reg = fastEmit_i_Predicate_simm6_32b(VT, RetVT, Opcode, imm0))
14722 return Reg;
14723
14724 switch (Opcode) {
14725 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
14726 default: return Register();
14727 }
14728}
14729
14730// FastEmit functions for AArch64ISD::FMOV.
14731
14732Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(uint64_t imm0) {
14733 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
14734 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv4f16_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
14735 }
14736 return Register();
14737}
14738
14739Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(uint64_t imm0) {
14740 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
14741 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv8f16_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14742 }
14743 return Register();
14744}
14745
14746Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(uint64_t imm0) {
14747 if ((Subtarget->isNeonAvailable())) {
14748 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv2f32_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
14749 }
14750 return Register();
14751}
14752
14753Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(uint64_t imm0) {
14754 if ((Subtarget->isNeonAvailable())) {
14755 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv4f32_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14756 }
14757 return Register();
14758}
14759
14760Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(uint64_t imm0) {
14761 if ((Subtarget->isNeonAvailable())) {
14762 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv2f64_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14763 }
14764 return Register();
14765}
14766
14767Register fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
14768switch (RetVT.SimpleTy) {
14769 case MVT::v4f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(imm0);
14770 case MVT::v8f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(imm0);
14771 case MVT::v2f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(imm0);
14772 case MVT::v4f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(imm0);
14773 case MVT::v2f64: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(imm0);
14774 default: return Register();
14775}
14776}
14777
14778Register fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
14779 switch (VT.SimpleTy) {
14780 case MVT::i32: return fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
14781 default: return Register();
14782 }
14783}
14784
14785// FastEmit functions for AArch64ISD::MOVI.
14786
14787Register fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(uint64_t imm0) {
14788 if ((Subtarget->isNeonAvailable())) {
14789 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv8b_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
14790 }
14791 return Register();
14792}
14793
14794Register fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(uint64_t imm0) {
14795 if ((Subtarget->isNeonAvailable())) {
14796 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv16b_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14797 }
14798 return Register();
14799}
14800
14801Register fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
14802switch (RetVT.SimpleTy) {
14803 case MVT::v8i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(imm0);
14804 case MVT::v16i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(imm0);
14805 default: return Register();
14806}
14807}
14808
14809Register fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
14810 switch (VT.SimpleTy) {
14811 case MVT::i32: return fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
14812 default: return Register();
14813 }
14814}
14815
14816// FastEmit functions for AArch64ISD::MOVIedit.
14817
14818Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(uint64_t imm0) {
14819 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVID, RC: &AArch64::FPR64RegClass, Imm: imm0);
14820}
14821
14822Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(uint64_t imm0) {
14823 if ((Subtarget->isNeonAvailable())) {
14824 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv2d_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14825 }
14826 return Register();
14827}
14828
14829Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
14830switch (RetVT.SimpleTy) {
14831 case MVT::f64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(imm0);
14832 case MVT::v2i64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(imm0);
14833 default: return Register();
14834}
14835}
14836
14837Register fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
14838 switch (VT.SimpleTy) {
14839 case MVT::i32: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
14840 default: return Register();
14841 }
14842}
14843
14844// Top-level FastEmit function.
14845
14846Register fastEmit_i_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
14847 switch (Opcode) {
14848 case AArch64ISD::FMOV: return fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(VT, RetVT, imm0);
14849 case AArch64ISD::MOVI: return fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(VT, RetVT, imm0);
14850 case AArch64ISD::MOVIedit: return fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(VT, RetVT, imm0);
14851 default: return Register();
14852 }
14853}
14854
14855// FastEmit functions for AArch64ISD::RDSVL.
14856
14857Register fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(MVT RetVT, uint64_t imm0) {
14858 if (RetVT.SimpleTy != MVT::i64)
14859 return Register();
14860 if ((Subtarget->hasSME())) {
14861 return fastEmitInst_i(MachineInstOpcode: AArch64::RDSVLI_XI, RC: &AArch64::GPR64RegClass, Imm: imm0);
14862 }
14863 return Register();
14864}
14865
14866Register fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(MVT VT, MVT RetVT, uint64_t imm0) {
14867 switch (VT.SimpleTy) {
14868 case MVT::i32: return fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(RetVT, imm0);
14869 default: return Register();
14870 }
14871}
14872
14873// Top-level FastEmit function.
14874
14875Register fastEmit_i_Predicate_simm6_32b(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
14876 switch (Opcode) {
14877 case AArch64ISD::RDSVL: return fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(VT, RetVT, imm0);
14878 default: return Register();
14879 }
14880}
14881
14882