1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the AArch64 target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_imm0_31(int64_t Imm) {
12
13 return ((uint64_t)Imm) < 32;
14
15}
16static bool Predicate_imm0_63(int64_t Imm) {
17
18 return ((uint64_t)Imm) < 64;
19
20}
21static bool Predicate_VectorIndexD(int64_t Imm) {
22 return ((uint64_t)Imm) < 2;
23}
24static bool Predicate_VectorIndexS(int64_t Imm) {
25 return ((uint64_t)Imm) < 4;
26}
27static bool Predicate_VectorIndexH(int64_t Imm) {
28 return ((uint64_t)Imm) < 8;
29}
30static bool Predicate_VectorIndexB(int64_t Imm) {
31 return ((uint64_t)Imm) < 16;
32}
33static bool Predicate_VectorIndex0(int64_t Imm) {
34 return ((uint64_t)Imm) == 0;
35}
36static bool Predicate_imm0_255(int64_t Imm) {
37
38 return ((uint32_t)Imm) < 256;
39
40}
41static bool Predicate_simm8_32b(int64_t Imm) {
42 return Imm >= -128 && Imm < 128;
43}
44static bool Predicate_simm8_64b(int64_t Imm) {
45 return Imm >= -128 && Imm < 128;
46}
47static bool Predicate_uimm8_32b(int64_t Imm) {
48 return Imm >= 0 && Imm < 256;
49}
50static bool Predicate_uimm8_64b(int64_t Imm) {
51 return Imm >= 0 && Imm < 256;
52}
53static bool Predicate_simm6_32b(int64_t Imm) {
54 return Imm >= -32 && Imm < 32;
55}
56
57
58// FastEmit functions for AArch64ISD::ENTRY_PSTATE_SM.
59
60Register fastEmit_AArch64ISD_ENTRY_PSTATE_SM_MVT_i64_(MVT RetVT) {
61 if (RetVT.SimpleTy != MVT::i64)
62 return Register();
63 return fastEmitInst_(MachineInstOpcode: AArch64::EntryPStateSM, RC: &AArch64::GPR64RegClass);
64}
65
66Register fastEmit_AArch64ISD_ENTRY_PSTATE_SM_(MVT VT, MVT RetVT) {
67 switch (VT.SimpleTy) {
68 case MVT::i64: return fastEmit_AArch64ISD_ENTRY_PSTATE_SM_MVT_i64_(RetVT);
69 default: return Register();
70 }
71}
72
73// FastEmit functions for AArch64ISD::THREAD_POINTER.
74
75Register fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(MVT RetVT) {
76 if (RetVT.SimpleTy != MVT::i64)
77 return Register();
78 return fastEmitInst_(MachineInstOpcode: AArch64::MOVbaseTLS, RC: &AArch64::GPR64RegClass);
79}
80
81Register fastEmit_AArch64ISD_THREAD_POINTER_(MVT VT, MVT RetVT) {
82 switch (VT.SimpleTy) {
83 case MVT::i64: return fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(RetVT);
84 default: return Register();
85 }
86}
87
88// Top-level FastEmit function.
89
90Register fastEmit_(MVT VT, MVT RetVT, unsigned Opcode) override {
91 switch (Opcode) {
92 case AArch64ISD::ENTRY_PSTATE_SM: return fastEmit_AArch64ISD_ENTRY_PSTATE_SM_(VT, RetVT);
93 case AArch64ISD::THREAD_POINTER: return fastEmit_AArch64ISD_THREAD_POINTER_(VT, RetVT);
94 default: return Register();
95 }
96}
97
98// FastEmit functions for AArch64ISD::COALESCER_BARRIER.
99
100Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_bf16_r(MVT RetVT, Register Op0) {
101 if (RetVT.SimpleTy != MVT::bf16)
102 return Register();
103 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR16, RC: &AArch64::FPR16RegClass, Op0);
104}
105
106Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f16_r(MVT RetVT, Register Op0) {
107 if (RetVT.SimpleTy != MVT::f16)
108 return Register();
109 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR16, RC: &AArch64::FPR16RegClass, Op0);
110}
111
112Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f32_r(MVT RetVT, Register Op0) {
113 if (RetVT.SimpleTy != MVT::f32)
114 return Register();
115 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR32, RC: &AArch64::FPR32RegClass, Op0);
116}
117
118Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f64_r(MVT RetVT, Register Op0) {
119 if (RetVT.SimpleTy != MVT::f64)
120 return Register();
121 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
122}
123
124Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f128_r(MVT RetVT, Register Op0) {
125 if (RetVT.SimpleTy != MVT::f128)
126 return Register();
127 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
128}
129
130Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i8_r(MVT RetVT, Register Op0) {
131 if (RetVT.SimpleTy != MVT::v8i8)
132 return Register();
133 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
134}
135
136Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v16i8_r(MVT RetVT, Register Op0) {
137 if (RetVT.SimpleTy != MVT::v16i8)
138 return Register();
139 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
140}
141
142Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i16_r(MVT RetVT, Register Op0) {
143 if (RetVT.SimpleTy != MVT::v4i16)
144 return Register();
145 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
146}
147
148Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i16_r(MVT RetVT, Register Op0) {
149 if (RetVT.SimpleTy != MVT::v8i16)
150 return Register();
151 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
152}
153
154Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i32_r(MVT RetVT, Register Op0) {
155 if (RetVT.SimpleTy != MVT::v2i32)
156 return Register();
157 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
158}
159
160Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i32_r(MVT RetVT, Register Op0) {
161 if (RetVT.SimpleTy != MVT::v4i32)
162 return Register();
163 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
164}
165
166Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1i64_r(MVT RetVT, Register Op0) {
167 if (RetVT.SimpleTy != MVT::v1i64)
168 return Register();
169 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
170}
171
172Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i64_r(MVT RetVT, Register Op0) {
173 if (RetVT.SimpleTy != MVT::v2i64)
174 return Register();
175 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
176}
177
178Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f16_r(MVT RetVT, Register Op0) {
179 if (RetVT.SimpleTy != MVT::v4f16)
180 return Register();
181 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
182}
183
184Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8f16_r(MVT RetVT, Register Op0) {
185 if (RetVT.SimpleTy != MVT::v8f16)
186 return Register();
187 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
188}
189
190Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4bf16_r(MVT RetVT, Register Op0) {
191 if (RetVT.SimpleTy != MVT::v4bf16)
192 return Register();
193 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
194}
195
196Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8bf16_r(MVT RetVT, Register Op0) {
197 if (RetVT.SimpleTy != MVT::v8bf16)
198 return Register();
199 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
200}
201
202Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f32_r(MVT RetVT, Register Op0) {
203 if (RetVT.SimpleTy != MVT::v2f32)
204 return Register();
205 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
206}
207
208Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f32_r(MVT RetVT, Register Op0) {
209 if (RetVT.SimpleTy != MVT::v4f32)
210 return Register();
211 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
212}
213
214Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1f64_r(MVT RetVT, Register Op0) {
215 if (RetVT.SimpleTy != MVT::v1f64)
216 return Register();
217 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
218}
219
220Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f64_r(MVT RetVT, Register Op0) {
221 if (RetVT.SimpleTy != MVT::v2f64)
222 return Register();
223 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
224}
225
226Register fastEmit_AArch64ISD_COALESCER_BARRIER_r(MVT VT, MVT RetVT, Register Op0) {
227 switch (VT.SimpleTy) {
228 case MVT::bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_bf16_r(RetVT, Op0);
229 case MVT::f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f16_r(RetVT, Op0);
230 case MVT::f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f32_r(RetVT, Op0);
231 case MVT::f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f64_r(RetVT, Op0);
232 case MVT::f128: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f128_r(RetVT, Op0);
233 case MVT::v8i8: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i8_r(RetVT, Op0);
234 case MVT::v16i8: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v16i8_r(RetVT, Op0);
235 case MVT::v4i16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i16_r(RetVT, Op0);
236 case MVT::v8i16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i16_r(RetVT, Op0);
237 case MVT::v2i32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i32_r(RetVT, Op0);
238 case MVT::v4i32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i32_r(RetVT, Op0);
239 case MVT::v1i64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1i64_r(RetVT, Op0);
240 case MVT::v2i64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i64_r(RetVT, Op0);
241 case MVT::v4f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f16_r(RetVT, Op0);
242 case MVT::v8f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8f16_r(RetVT, Op0);
243 case MVT::v4bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4bf16_r(RetVT, Op0);
244 case MVT::v8bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8bf16_r(RetVT, Op0);
245 case MVT::v2f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f32_r(RetVT, Op0);
246 case MVT::v4f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f32_r(RetVT, Op0);
247 case MVT::v1f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1f64_r(RetVT, Op0);
248 case MVT::v2f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f64_r(RetVT, Op0);
249 default: return Register();
250 }
251}
252
253// FastEmit functions for AArch64ISD::DUP.
254
255Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Register Op0) {
256 if ((Subtarget->isNeonAvailable())) {
257 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv8i8gpr, RC: &AArch64::FPR64RegClass, Op0);
258 }
259 return Register();
260}
261
262Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Register Op0) {
263 if ((Subtarget->isNeonAvailable())) {
264 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv16i8gpr, RC: &AArch64::FPR128RegClass, Op0);
265 }
266 return Register();
267}
268
269Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Register Op0) {
270 if ((Subtarget->isNeonAvailable())) {
271 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv4i16gpr, RC: &AArch64::FPR64RegClass, Op0);
272 }
273 return Register();
274}
275
276Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Register Op0) {
277 if ((Subtarget->isNeonAvailable())) {
278 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv8i16gpr, RC: &AArch64::FPR128RegClass, Op0);
279 }
280 return Register();
281}
282
283Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Register Op0) {
284 if ((Subtarget->isNeonAvailable())) {
285 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv2i32gpr, RC: &AArch64::FPR64RegClass, Op0);
286 }
287 return Register();
288}
289
290Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Register Op0) {
291 if ((Subtarget->isNeonAvailable())) {
292 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv4i32gpr, RC: &AArch64::FPR128RegClass, Op0);
293 }
294 return Register();
295}
296
297Register fastEmit_AArch64ISD_DUP_MVT_i32_r(MVT RetVT, Register Op0) {
298switch (RetVT.SimpleTy) {
299 case MVT::v8i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Op0);
300 case MVT::v16i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Op0);
301 case MVT::v4i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Op0);
302 case MVT::v8i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Op0);
303 case MVT::v2i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Op0);
304 case MVT::v4i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Op0);
305 default: return Register();
306}
307}
308
309Register fastEmit_AArch64ISD_DUP_MVT_i64_r(MVT RetVT, Register Op0) {
310 if (RetVT.SimpleTy != MVT::v2i64)
311 return Register();
312 if ((Subtarget->isNeonAvailable())) {
313 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv2i64gpr, RC: &AArch64::FPR128RegClass, Op0);
314 }
315 return Register();
316}
317
318Register fastEmit_AArch64ISD_DUP_r(MVT VT, MVT RetVT, Register Op0) {
319 switch (VT.SimpleTy) {
320 case MVT::i32: return fastEmit_AArch64ISD_DUP_MVT_i32_r(RetVT, Op0);
321 case MVT::i64: return fastEmit_AArch64ISD_DUP_MVT_i64_r(RetVT, Op0);
322 default: return Register();
323 }
324}
325
326// FastEmit functions for AArch64ISD::FCVTXN.
327
328Register fastEmit_AArch64ISD_FCVTXN_MVT_f64_r(MVT RetVT, Register Op0) {
329 if (RetVT.SimpleTy != MVT::f32)
330 return Register();
331 if ((Subtarget->isNeonAvailable())) {
332 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTXNv1i64, RC: &AArch64::FPR32RegClass, Op0);
333 }
334 return Register();
335}
336
337Register fastEmit_AArch64ISD_FCVTXN_MVT_v2f64_r(MVT RetVT, Register Op0) {
338 if (RetVT.SimpleTy != MVT::v2f32)
339 return Register();
340 if ((Subtarget->isNeonAvailable())) {
341 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTXNv2f32, RC: &AArch64::FPR64RegClass, Op0);
342 }
343 return Register();
344}
345
346Register fastEmit_AArch64ISD_FCVTXN_r(MVT VT, MVT RetVT, Register Op0) {
347 switch (VT.SimpleTy) {
348 case MVT::f64: return fastEmit_AArch64ISD_FCVTXN_MVT_f64_r(RetVT, Op0);
349 case MVT::v2f64: return fastEmit_AArch64ISD_FCVTXN_MVT_v2f64_r(RetVT, Op0);
350 default: return Register();
351 }
352}
353
354// FastEmit functions for AArch64ISD::FRECPE.
355
356Register fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(MVT RetVT, Register Op0) {
357 if (RetVT.SimpleTy != MVT::v2f32)
358 return Register();
359 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv2f32, RC: &AArch64::FPR64RegClass, Op0);
360}
361
362Register fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(MVT RetVT, Register Op0) {
363 if (RetVT.SimpleTy != MVT::v4f32)
364 return Register();
365 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv4f32, RC: &AArch64::FPR128RegClass, Op0);
366}
367
368Register fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(MVT RetVT, Register Op0) {
369 if (RetVT.SimpleTy != MVT::v2f64)
370 return Register();
371 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv2f64, RC: &AArch64::FPR128RegClass, Op0);
372}
373
374Register fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
375 if (RetVT.SimpleTy != MVT::nxv8f16)
376 return Register();
377 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
378 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
379 }
380 return Register();
381}
382
383Register fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
384 if (RetVT.SimpleTy != MVT::nxv4f32)
385 return Register();
386 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
387 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
388 }
389 return Register();
390}
391
392Register fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
393 if (RetVT.SimpleTy != MVT::nxv2f64)
394 return Register();
395 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
396 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
397 }
398 return Register();
399}
400
401Register fastEmit_AArch64ISD_FRECPE_r(MVT VT, MVT RetVT, Register Op0) {
402 switch (VT.SimpleTy) {
403 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(RetVT, Op0);
404 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(RetVT, Op0);
405 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(RetVT, Op0);
406 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(RetVT, Op0);
407 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(RetVT, Op0);
408 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(RetVT, Op0);
409 default: return Register();
410 }
411}
412
413// FastEmit functions for AArch64ISD::FRSQRTE.
414
415Register fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(MVT RetVT, Register Op0) {
416 if (RetVT.SimpleTy != MVT::v2f32)
417 return Register();
418 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv2f32, RC: &AArch64::FPR64RegClass, Op0);
419}
420
421Register fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(MVT RetVT, Register Op0) {
422 if (RetVT.SimpleTy != MVT::v4f32)
423 return Register();
424 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv4f32, RC: &AArch64::FPR128RegClass, Op0);
425}
426
427Register fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(MVT RetVT, Register Op0) {
428 if (RetVT.SimpleTy != MVT::v2f64)
429 return Register();
430 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv2f64, RC: &AArch64::FPR128RegClass, Op0);
431}
432
433Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
434 if (RetVT.SimpleTy != MVT::nxv8f16)
435 return Register();
436 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
437 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
438 }
439 return Register();
440}
441
442Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
443 if (RetVT.SimpleTy != MVT::nxv4f32)
444 return Register();
445 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
446 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
447 }
448 return Register();
449}
450
451Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
452 if (RetVT.SimpleTy != MVT::nxv2f64)
453 return Register();
454 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
455 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
456 }
457 return Register();
458}
459
460Register fastEmit_AArch64ISD_FRSQRTE_r(MVT VT, MVT RetVT, Register Op0) {
461 switch (VT.SimpleTy) {
462 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(RetVT, Op0);
463 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(RetVT, Op0);
464 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(RetVT, Op0);
465 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(RetVT, Op0);
466 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(RetVT, Op0);
467 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(RetVT, Op0);
468 default: return Register();
469 }
470}
471
472// FastEmit functions for AArch64ISD::REV16.
473
474Register fastEmit_AArch64ISD_REV16_MVT_i32_r(MVT RetVT, Register Op0) {
475 if (RetVT.SimpleTy != MVT::i32)
476 return Register();
477 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16Wr, RC: &AArch64::GPR32RegClass, Op0);
478}
479
480Register fastEmit_AArch64ISD_REV16_MVT_i64_r(MVT RetVT, Register Op0) {
481 if (RetVT.SimpleTy != MVT::i64)
482 return Register();
483 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16Xr, RC: &AArch64::GPR64RegClass, Op0);
484}
485
486Register fastEmit_AArch64ISD_REV16_r(MVT VT, MVT RetVT, Register Op0) {
487 switch (VT.SimpleTy) {
488 case MVT::i32: return fastEmit_AArch64ISD_REV16_MVT_i32_r(RetVT, Op0);
489 case MVT::i64: return fastEmit_AArch64ISD_REV16_MVT_i64_r(RetVT, Op0);
490 default: return Register();
491 }
492}
493
494// FastEmit functions for AArch64ISD::REV32.
495
496Register fastEmit_AArch64ISD_REV32_MVT_v8i8_r(MVT RetVT, Register Op0) {
497 if (RetVT.SimpleTy != MVT::v8i8)
498 return Register();
499 if ((Subtarget->isNeonAvailable())) {
500 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
501 }
502 return Register();
503}
504
505Register fastEmit_AArch64ISD_REV32_MVT_v16i8_r(MVT RetVT, Register Op0) {
506 if (RetVT.SimpleTy != MVT::v16i8)
507 return Register();
508 if ((Subtarget->isNeonAvailable())) {
509 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
510 }
511 return Register();
512}
513
514Register fastEmit_AArch64ISD_REV32_MVT_v4i16_r(MVT RetVT, Register Op0) {
515 if (RetVT.SimpleTy != MVT::v4i16)
516 return Register();
517 if ((Subtarget->isNeonAvailable())) {
518 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
519 }
520 return Register();
521}
522
523Register fastEmit_AArch64ISD_REV32_MVT_v8i16_r(MVT RetVT, Register Op0) {
524 if (RetVT.SimpleTy != MVT::v8i16)
525 return Register();
526 if ((Subtarget->isNeonAvailable())) {
527 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
528 }
529 return Register();
530}
531
532Register fastEmit_AArch64ISD_REV32_MVT_v4f16_r(MVT RetVT, Register Op0) {
533 if (RetVT.SimpleTy != MVT::v4f16)
534 return Register();
535 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
536}
537
538Register fastEmit_AArch64ISD_REV32_MVT_v8f16_r(MVT RetVT, Register Op0) {
539 if (RetVT.SimpleTy != MVT::v8f16)
540 return Register();
541 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
542}
543
544Register fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(MVT RetVT, Register Op0) {
545 if (RetVT.SimpleTy != MVT::v4bf16)
546 return Register();
547 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
548}
549
550Register fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(MVT RetVT, Register Op0) {
551 if (RetVT.SimpleTy != MVT::v8bf16)
552 return Register();
553 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
554}
555
556Register fastEmit_AArch64ISD_REV32_r(MVT VT, MVT RetVT, Register Op0) {
557 switch (VT.SimpleTy) {
558 case MVT::v8i8: return fastEmit_AArch64ISD_REV32_MVT_v8i8_r(RetVT, Op0);
559 case MVT::v16i8: return fastEmit_AArch64ISD_REV32_MVT_v16i8_r(RetVT, Op0);
560 case MVT::v4i16: return fastEmit_AArch64ISD_REV32_MVT_v4i16_r(RetVT, Op0);
561 case MVT::v8i16: return fastEmit_AArch64ISD_REV32_MVT_v8i16_r(RetVT, Op0);
562 case MVT::v4f16: return fastEmit_AArch64ISD_REV32_MVT_v4f16_r(RetVT, Op0);
563 case MVT::v8f16: return fastEmit_AArch64ISD_REV32_MVT_v8f16_r(RetVT, Op0);
564 case MVT::v4bf16: return fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(RetVT, Op0);
565 case MVT::v8bf16: return fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(RetVT, Op0);
566 default: return Register();
567 }
568}
569
570// FastEmit functions for AArch64ISD::REV64.
571
572Register fastEmit_AArch64ISD_REV64_MVT_v8i8_r(MVT RetVT, Register Op0) {
573 if (RetVT.SimpleTy != MVT::v8i8)
574 return Register();
575 if ((Subtarget->isNeonAvailable())) {
576 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
577 }
578 return Register();
579}
580
581Register fastEmit_AArch64ISD_REV64_MVT_v16i8_r(MVT RetVT, Register Op0) {
582 if (RetVT.SimpleTy != MVT::v16i8)
583 return Register();
584 if ((Subtarget->isNeonAvailable())) {
585 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
586 }
587 return Register();
588}
589
590Register fastEmit_AArch64ISD_REV64_MVT_v4i16_r(MVT RetVT, Register Op0) {
591 if (RetVT.SimpleTy != MVT::v4i16)
592 return Register();
593 if ((Subtarget->isNeonAvailable())) {
594 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
595 }
596 return Register();
597}
598
599Register fastEmit_AArch64ISD_REV64_MVT_v8i16_r(MVT RetVT, Register Op0) {
600 if (RetVT.SimpleTy != MVT::v8i16)
601 return Register();
602 if ((Subtarget->isNeonAvailable())) {
603 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
604 }
605 return Register();
606}
607
608Register fastEmit_AArch64ISD_REV64_MVT_v2i32_r(MVT RetVT, Register Op0) {
609 if (RetVT.SimpleTy != MVT::v2i32)
610 return Register();
611 if ((Subtarget->isNeonAvailable())) {
612 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
613 }
614 return Register();
615}
616
617Register fastEmit_AArch64ISD_REV64_MVT_v4i32_r(MVT RetVT, Register Op0) {
618 if (RetVT.SimpleTy != MVT::v4i32)
619 return Register();
620 if ((Subtarget->isNeonAvailable())) {
621 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
622 }
623 return Register();
624}
625
626Register fastEmit_AArch64ISD_REV64_MVT_v4f16_r(MVT RetVT, Register Op0) {
627 if (RetVT.SimpleTy != MVT::v4f16)
628 return Register();
629 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
630}
631
632Register fastEmit_AArch64ISD_REV64_MVT_v8f16_r(MVT RetVT, Register Op0) {
633 if (RetVT.SimpleTy != MVT::v8f16)
634 return Register();
635 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
636}
637
638Register fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(MVT RetVT, Register Op0) {
639 if (RetVT.SimpleTy != MVT::v4bf16)
640 return Register();
641 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
642}
643
644Register fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(MVT RetVT, Register Op0) {
645 if (RetVT.SimpleTy != MVT::v8bf16)
646 return Register();
647 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
648}
649
650Register fastEmit_AArch64ISD_REV64_MVT_v2f32_r(MVT RetVT, Register Op0) {
651 if (RetVT.SimpleTy != MVT::v2f32)
652 return Register();
653 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
654}
655
656Register fastEmit_AArch64ISD_REV64_MVT_v4f32_r(MVT RetVT, Register Op0) {
657 if (RetVT.SimpleTy != MVT::v4f32)
658 return Register();
659 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
660}
661
662Register fastEmit_AArch64ISD_REV64_r(MVT VT, MVT RetVT, Register Op0) {
663 switch (VT.SimpleTy) {
664 case MVT::v8i8: return fastEmit_AArch64ISD_REV64_MVT_v8i8_r(RetVT, Op0);
665 case MVT::v16i8: return fastEmit_AArch64ISD_REV64_MVT_v16i8_r(RetVT, Op0);
666 case MVT::v4i16: return fastEmit_AArch64ISD_REV64_MVT_v4i16_r(RetVT, Op0);
667 case MVT::v8i16: return fastEmit_AArch64ISD_REV64_MVT_v8i16_r(RetVT, Op0);
668 case MVT::v2i32: return fastEmit_AArch64ISD_REV64_MVT_v2i32_r(RetVT, Op0);
669 case MVT::v4i32: return fastEmit_AArch64ISD_REV64_MVT_v4i32_r(RetVT, Op0);
670 case MVT::v4f16: return fastEmit_AArch64ISD_REV64_MVT_v4f16_r(RetVT, Op0);
671 case MVT::v8f16: return fastEmit_AArch64ISD_REV64_MVT_v8f16_r(RetVT, Op0);
672 case MVT::v4bf16: return fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(RetVT, Op0);
673 case MVT::v8bf16: return fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(RetVT, Op0);
674 case MVT::v2f32: return fastEmit_AArch64ISD_REV64_MVT_v2f32_r(RetVT, Op0);
675 case MVT::v4f32: return fastEmit_AArch64ISD_REV64_MVT_v4f32_r(RetVT, Op0);
676 default: return Register();
677 }
678}
679
680// FastEmit functions for AArch64ISD::SADDLP.
681
682Register fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(MVT RetVT, Register Op0) {
683 if (RetVT.SimpleTy != MVT::v4i16)
684 return Register();
685 if ((Subtarget->isNeonAvailable())) {
686 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv8i8_v4i16, RC: &AArch64::FPR64RegClass, Op0);
687 }
688 return Register();
689}
690
691Register fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(MVT RetVT, Register Op0) {
692 if (RetVT.SimpleTy != MVT::v8i16)
693 return Register();
694 if ((Subtarget->isNeonAvailable())) {
695 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv16i8_v8i16, RC: &AArch64::FPR128RegClass, Op0);
696 }
697 return Register();
698}
699
700Register fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(MVT RetVT, Register Op0) {
701 if (RetVT.SimpleTy != MVT::v2i32)
702 return Register();
703 if ((Subtarget->isNeonAvailable())) {
704 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv4i16_v2i32, RC: &AArch64::FPR64RegClass, Op0);
705 }
706 return Register();
707}
708
709Register fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(MVT RetVT, Register Op0) {
710 if (RetVT.SimpleTy != MVT::v4i32)
711 return Register();
712 if ((Subtarget->isNeonAvailable())) {
713 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv8i16_v4i32, RC: &AArch64::FPR128RegClass, Op0);
714 }
715 return Register();
716}
717
718Register fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(MVT RetVT, Register Op0) {
719 if (RetVT.SimpleTy != MVT::v1i64)
720 return Register();
721 if ((Subtarget->isNeonAvailable())) {
722 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv2i32_v1i64, RC: &AArch64::FPR64RegClass, Op0);
723 }
724 return Register();
725}
726
727Register fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(MVT RetVT, Register Op0) {
728 if (RetVT.SimpleTy != MVT::v2i64)
729 return Register();
730 if ((Subtarget->isNeonAvailable())) {
731 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv4i32_v2i64, RC: &AArch64::FPR128RegClass, Op0);
732 }
733 return Register();
734}
735
736Register fastEmit_AArch64ISD_SADDLP_r(MVT VT, MVT RetVT, Register Op0) {
737 switch (VT.SimpleTy) {
738 case MVT::v8i8: return fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(RetVT, Op0);
739 case MVT::v16i8: return fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(RetVT, Op0);
740 case MVT::v4i16: return fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(RetVT, Op0);
741 case MVT::v8i16: return fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(RetVT, Op0);
742 case MVT::v2i32: return fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(RetVT, Op0);
743 case MVT::v4i32: return fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(RetVT, Op0);
744 default: return Register();
745 }
746}
747
748// FastEmit functions for AArch64ISD::SITOF.
749
750Register fastEmit_AArch64ISD_SITOF_MVT_f16_r(MVT RetVT, Register Op0) {
751 if (RetVT.SimpleTy != MVT::f16)
752 return Register();
753 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
754 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i16, RC: &AArch64::FPR16RegClass, Op0);
755 }
756 return Register();
757}
758
759Register fastEmit_AArch64ISD_SITOF_MVT_f32_r(MVT RetVT, Register Op0) {
760 if (RetVT.SimpleTy != MVT::f32)
761 return Register();
762 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
763 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i32, RC: &AArch64::FPR32RegClass, Op0);
764 }
765 return Register();
766}
767
768Register fastEmit_AArch64ISD_SITOF_MVT_f64_r(MVT RetVT, Register Op0) {
769 if (RetVT.SimpleTy != MVT::f64)
770 return Register();
771 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
772 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i64, RC: &AArch64::FPR64RegClass, Op0);
773 }
774 return Register();
775}
776
777Register fastEmit_AArch64ISD_SITOF_r(MVT VT, MVT RetVT, Register Op0) {
778 switch (VT.SimpleTy) {
779 case MVT::f16: return fastEmit_AArch64ISD_SITOF_MVT_f16_r(RetVT, Op0);
780 case MVT::f32: return fastEmit_AArch64ISD_SITOF_MVT_f32_r(RetVT, Op0);
781 case MVT::f64: return fastEmit_AArch64ISD_SITOF_MVT_f64_r(RetVT, Op0);
782 default: return Register();
783 }
784}
785
786// FastEmit functions for AArch64ISD::SQABS.
787
788Register fastEmit_AArch64ISD_SQABS_MVT_f32_r(MVT RetVT, Register Op0) {
789 if (RetVT.SimpleTy != MVT::f32)
790 return Register();
791 if ((Subtarget->isNeonAvailable())) {
792 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv1i32, RC: &AArch64::FPR32RegClass, Op0);
793 }
794 return Register();
795}
796
797Register fastEmit_AArch64ISD_SQABS_MVT_f64_r(MVT RetVT, Register Op0) {
798 if (RetVT.SimpleTy != MVT::f64)
799 return Register();
800 if ((Subtarget->isNeonAvailable())) {
801 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
802 }
803 return Register();
804}
805
806Register fastEmit_AArch64ISD_SQABS_MVT_v8i8_r(MVT RetVT, Register Op0) {
807 if (RetVT.SimpleTy != MVT::v8i8)
808 return Register();
809 if ((Subtarget->isNeonAvailable())) {
810 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv8i8, RC: &AArch64::FPR64RegClass, Op0);
811 }
812 return Register();
813}
814
815Register fastEmit_AArch64ISD_SQABS_MVT_v16i8_r(MVT RetVT, Register Op0) {
816 if (RetVT.SimpleTy != MVT::v16i8)
817 return Register();
818 if ((Subtarget->isNeonAvailable())) {
819 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv16i8, RC: &AArch64::FPR128RegClass, Op0);
820 }
821 return Register();
822}
823
824Register fastEmit_AArch64ISD_SQABS_MVT_v4i16_r(MVT RetVT, Register Op0) {
825 if (RetVT.SimpleTy != MVT::v4i16)
826 return Register();
827 if ((Subtarget->isNeonAvailable())) {
828 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv4i16, RC: &AArch64::FPR64RegClass, Op0);
829 }
830 return Register();
831}
832
833Register fastEmit_AArch64ISD_SQABS_MVT_v8i16_r(MVT RetVT, Register Op0) {
834 if (RetVT.SimpleTy != MVT::v8i16)
835 return Register();
836 if ((Subtarget->isNeonAvailable())) {
837 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv8i16, RC: &AArch64::FPR128RegClass, Op0);
838 }
839 return Register();
840}
841
842Register fastEmit_AArch64ISD_SQABS_MVT_v2i32_r(MVT RetVT, Register Op0) {
843 if (RetVT.SimpleTy != MVT::v2i32)
844 return Register();
845 if ((Subtarget->isNeonAvailable())) {
846 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv2i32, RC: &AArch64::FPR64RegClass, Op0);
847 }
848 return Register();
849}
850
851Register fastEmit_AArch64ISD_SQABS_MVT_v4i32_r(MVT RetVT, Register Op0) {
852 if (RetVT.SimpleTy != MVT::v4i32)
853 return Register();
854 if ((Subtarget->isNeonAvailable())) {
855 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv4i32, RC: &AArch64::FPR128RegClass, Op0);
856 }
857 return Register();
858}
859
860Register fastEmit_AArch64ISD_SQABS_MVT_v1i64_r(MVT RetVT, Register Op0) {
861 if (RetVT.SimpleTy != MVT::v1i64)
862 return Register();
863 if ((Subtarget->isNeonAvailable())) {
864 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
865 }
866 return Register();
867}
868
869Register fastEmit_AArch64ISD_SQABS_MVT_v2i64_r(MVT RetVT, Register Op0) {
870 if (RetVT.SimpleTy != MVT::v2i64)
871 return Register();
872 if ((Subtarget->isNeonAvailable())) {
873 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv2i64, RC: &AArch64::FPR128RegClass, Op0);
874 }
875 return Register();
876}
877
878Register fastEmit_AArch64ISD_SQABS_r(MVT VT, MVT RetVT, Register Op0) {
879 switch (VT.SimpleTy) {
880 case MVT::f32: return fastEmit_AArch64ISD_SQABS_MVT_f32_r(RetVT, Op0);
881 case MVT::f64: return fastEmit_AArch64ISD_SQABS_MVT_f64_r(RetVT, Op0);
882 case MVT::v8i8: return fastEmit_AArch64ISD_SQABS_MVT_v8i8_r(RetVT, Op0);
883 case MVT::v16i8: return fastEmit_AArch64ISD_SQABS_MVT_v16i8_r(RetVT, Op0);
884 case MVT::v4i16: return fastEmit_AArch64ISD_SQABS_MVT_v4i16_r(RetVT, Op0);
885 case MVT::v8i16: return fastEmit_AArch64ISD_SQABS_MVT_v8i16_r(RetVT, Op0);
886 case MVT::v2i32: return fastEmit_AArch64ISD_SQABS_MVT_v2i32_r(RetVT, Op0);
887 case MVT::v4i32: return fastEmit_AArch64ISD_SQABS_MVT_v4i32_r(RetVT, Op0);
888 case MVT::v1i64: return fastEmit_AArch64ISD_SQABS_MVT_v1i64_r(RetVT, Op0);
889 case MVT::v2i64: return fastEmit_AArch64ISD_SQABS_MVT_v2i64_r(RetVT, Op0);
890 default: return Register();
891 }
892}
893
894// FastEmit functions for AArch64ISD::SQNEG.
895
896Register fastEmit_AArch64ISD_SQNEG_MVT_f32_r(MVT RetVT, Register Op0) {
897 if (RetVT.SimpleTy != MVT::f32)
898 return Register();
899 if ((Subtarget->isNeonAvailable())) {
900 return fastEmitInst_r(MachineInstOpcode: AArch64::SQNEGv1i32, RC: &AArch64::FPR32RegClass, Op0);
901 }
902 return Register();
903}
904
905Register fastEmit_AArch64ISD_SQNEG_MVT_f64_r(MVT RetVT, Register Op0) {
906 if (RetVT.SimpleTy != MVT::f64)
907 return Register();
908 if ((Subtarget->isNeonAvailable())) {
909 return fastEmitInst_r(MachineInstOpcode: AArch64::SQNEGv1i64, RC: &AArch64::FPR64RegClass, Op0);
910 }
911 return Register();
912}
913
914Register fastEmit_AArch64ISD_SQNEG_r(MVT VT, MVT RetVT, Register Op0) {
915 switch (VT.SimpleTy) {
916 case MVT::f32: return fastEmit_AArch64ISD_SQNEG_MVT_f32_r(RetVT, Op0);
917 case MVT::f64: return fastEmit_AArch64ISD_SQNEG_MVT_f64_r(RetVT, Op0);
918 default: return Register();
919 }
920}
921
922// FastEmit functions for AArch64ISD::SUNPKHI.
923
924Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
925 if (RetVT.SimpleTy != MVT::nxv8i16)
926 return Register();
927 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
928 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
929 }
930 return Register();
931}
932
933Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
934 if (RetVT.SimpleTy != MVT::nxv4i32)
935 return Register();
936 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
937 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
938 }
939 return Register();
940}
941
942Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
943 if (RetVT.SimpleTy != MVT::nxv2i64)
944 return Register();
945 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
946 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
947 }
948 return Register();
949}
950
951Register fastEmit_AArch64ISD_SUNPKHI_r(MVT VT, MVT RetVT, Register Op0) {
952 switch (VT.SimpleTy) {
953 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(RetVT, Op0);
954 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(RetVT, Op0);
955 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(RetVT, Op0);
956 default: return Register();
957 }
958}
959
960// FastEmit functions for AArch64ISD::SUNPKLO.
961
962Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
963 if (RetVT.SimpleTy != MVT::nxv8i16)
964 return Register();
965 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
966 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
967 }
968 return Register();
969}
970
971Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
972 if (RetVT.SimpleTy != MVT::nxv4i32)
973 return Register();
974 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
975 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
976 }
977 return Register();
978}
979
980Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
981 if (RetVT.SimpleTy != MVT::nxv2i64)
982 return Register();
983 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
984 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
985 }
986 return Register();
987}
988
989Register fastEmit_AArch64ISD_SUNPKLO_r(MVT VT, MVT RetVT, Register Op0) {
990 switch (VT.SimpleTy) {
991 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(RetVT, Op0);
992 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(RetVT, Op0);
993 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(RetVT, Op0);
994 default: return Register();
995 }
996}
997
998// FastEmit functions for AArch64ISD::UADDLP.
999
1000Register fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(MVT RetVT, Register Op0) {
1001 if (RetVT.SimpleTy != MVT::v4i16)
1002 return Register();
1003 if ((Subtarget->isNeonAvailable())) {
1004 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv8i8_v4i16, RC: &AArch64::FPR64RegClass, Op0);
1005 }
1006 return Register();
1007}
1008
1009Register fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(MVT RetVT, Register Op0) {
1010 if (RetVT.SimpleTy != MVT::v8i16)
1011 return Register();
1012 if ((Subtarget->isNeonAvailable())) {
1013 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv16i8_v8i16, RC: &AArch64::FPR128RegClass, Op0);
1014 }
1015 return Register();
1016}
1017
1018Register fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(MVT RetVT, Register Op0) {
1019 if (RetVT.SimpleTy != MVT::v2i32)
1020 return Register();
1021 if ((Subtarget->isNeonAvailable())) {
1022 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv4i16_v2i32, RC: &AArch64::FPR64RegClass, Op0);
1023 }
1024 return Register();
1025}
1026
1027Register fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(MVT RetVT, Register Op0) {
1028 if (RetVT.SimpleTy != MVT::v4i32)
1029 return Register();
1030 if ((Subtarget->isNeonAvailable())) {
1031 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv8i16_v4i32, RC: &AArch64::FPR128RegClass, Op0);
1032 }
1033 return Register();
1034}
1035
1036Register fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(MVT RetVT, Register Op0) {
1037 if (RetVT.SimpleTy != MVT::v1i64)
1038 return Register();
1039 if ((Subtarget->isNeonAvailable())) {
1040 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv2i32_v1i64, RC: &AArch64::FPR64RegClass, Op0);
1041 }
1042 return Register();
1043}
1044
1045Register fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(MVT RetVT, Register Op0) {
1046 if (RetVT.SimpleTy != MVT::v2i64)
1047 return Register();
1048 if ((Subtarget->isNeonAvailable())) {
1049 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv4i32_v2i64, RC: &AArch64::FPR128RegClass, Op0);
1050 }
1051 return Register();
1052}
1053
1054Register fastEmit_AArch64ISD_UADDLP_r(MVT VT, MVT RetVT, Register Op0) {
1055 switch (VT.SimpleTy) {
1056 case MVT::v8i8: return fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(RetVT, Op0);
1057 case MVT::v16i8: return fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(RetVT, Op0);
1058 case MVT::v4i16: return fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(RetVT, Op0);
1059 case MVT::v8i16: return fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(RetVT, Op0);
1060 case MVT::v2i32: return fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(RetVT, Op0);
1061 case MVT::v4i32: return fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(RetVT, Op0);
1062 default: return Register();
1063 }
1064}
1065
1066// FastEmit functions for AArch64ISD::UITOF.
1067
1068Register fastEmit_AArch64ISD_UITOF_MVT_f16_r(MVT RetVT, Register Op0) {
1069 if (RetVT.SimpleTy != MVT::f16)
1070 return Register();
1071 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
1072 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i16, RC: &AArch64::FPR16RegClass, Op0);
1073 }
1074 return Register();
1075}
1076
1077Register fastEmit_AArch64ISD_UITOF_MVT_f32_r(MVT RetVT, Register Op0) {
1078 if (RetVT.SimpleTy != MVT::f32)
1079 return Register();
1080 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
1081 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i32, RC: &AArch64::FPR32RegClass, Op0);
1082 }
1083 return Register();
1084}
1085
1086Register fastEmit_AArch64ISD_UITOF_MVT_f64_r(MVT RetVT, Register Op0) {
1087 if (RetVT.SimpleTy != MVT::f64)
1088 return Register();
1089 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
1090 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i64, RC: &AArch64::FPR64RegClass, Op0);
1091 }
1092 return Register();
1093}
1094
1095Register fastEmit_AArch64ISD_UITOF_r(MVT VT, MVT RetVT, Register Op0) {
1096 switch (VT.SimpleTy) {
1097 case MVT::f16: return fastEmit_AArch64ISD_UITOF_MVT_f16_r(RetVT, Op0);
1098 case MVT::f32: return fastEmit_AArch64ISD_UITOF_MVT_f32_r(RetVT, Op0);
1099 case MVT::f64: return fastEmit_AArch64ISD_UITOF_MVT_f64_r(RetVT, Op0);
1100 default: return Register();
1101 }
1102}
1103
1104// FastEmit functions for AArch64ISD::UUNPKHI.
1105
1106Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
1107 if (RetVT.SimpleTy != MVT::nxv8i16)
1108 return Register();
1109 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1110 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1111 }
1112 return Register();
1113}
1114
1115Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
1116 if (RetVT.SimpleTy != MVT::nxv4i32)
1117 return Register();
1118 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1119 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1120 }
1121 return Register();
1122}
1123
1124Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
1125 if (RetVT.SimpleTy != MVT::nxv2i64)
1126 return Register();
1127 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1128 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1129 }
1130 return Register();
1131}
1132
1133Register fastEmit_AArch64ISD_UUNPKHI_r(MVT VT, MVT RetVT, Register Op0) {
1134 switch (VT.SimpleTy) {
1135 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(RetVT, Op0);
1136 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(RetVT, Op0);
1137 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(RetVT, Op0);
1138 default: return Register();
1139 }
1140}
1141
1142// FastEmit functions for AArch64ISD::UUNPKLO.
1143
1144Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
1145 if (RetVT.SimpleTy != MVT::nxv8i16)
1146 return Register();
1147 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1148 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1149 }
1150 return Register();
1151}
1152
1153Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
1154 if (RetVT.SimpleTy != MVT::nxv4i32)
1155 return Register();
1156 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1157 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1158 }
1159 return Register();
1160}
1161
1162Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
1163 if (RetVT.SimpleTy != MVT::nxv2i64)
1164 return Register();
1165 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1166 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1167 }
1168 return Register();
1169}
1170
1171Register fastEmit_AArch64ISD_UUNPKLO_r(MVT VT, MVT RetVT, Register Op0) {
1172 switch (VT.SimpleTy) {
1173 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(RetVT, Op0);
1174 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(RetVT, Op0);
1175 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(RetVT, Op0);
1176 default: return Register();
1177 }
1178}
1179
1180// FastEmit functions for ISD::ABS.
1181
1182Register fastEmit_ISD_ABS_MVT_i32_r(MVT RetVT, Register Op0) {
1183 if (RetVT.SimpleTy != MVT::i32)
1184 return Register();
1185 if ((Subtarget->hasCSSC())) {
1186 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSWr, RC: &AArch64::GPR32RegClass, Op0);
1187 }
1188 return Register();
1189}
1190
1191Register fastEmit_ISD_ABS_MVT_i64_r(MVT RetVT, Register Op0) {
1192 if (RetVT.SimpleTy != MVT::i64)
1193 return Register();
1194 if ((!Subtarget->hasCSSC())) {
1195 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
1196 }
1197 if ((Subtarget->hasCSSC())) {
1198 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSXr, RC: &AArch64::GPR64RegClass, Op0);
1199 }
1200 return Register();
1201}
1202
1203Register fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, Register Op0) {
1204 if (RetVT.SimpleTy != MVT::v8i8)
1205 return Register();
1206 if ((Subtarget->isNeonAvailable())) {
1207 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv8i8, RC: &AArch64::FPR64RegClass, Op0);
1208 }
1209 return Register();
1210}
1211
1212Register fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, Register Op0) {
1213 if (RetVT.SimpleTy != MVT::v16i8)
1214 return Register();
1215 if ((Subtarget->isNeonAvailable())) {
1216 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv16i8, RC: &AArch64::FPR128RegClass, Op0);
1217 }
1218 return Register();
1219}
1220
1221Register fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, Register Op0) {
1222 if (RetVT.SimpleTy != MVT::v4i16)
1223 return Register();
1224 if ((Subtarget->isNeonAvailable())) {
1225 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv4i16, RC: &AArch64::FPR64RegClass, Op0);
1226 }
1227 return Register();
1228}
1229
1230Register fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, Register Op0) {
1231 if (RetVT.SimpleTy != MVT::v8i16)
1232 return Register();
1233 if ((Subtarget->isNeonAvailable())) {
1234 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv8i16, RC: &AArch64::FPR128RegClass, Op0);
1235 }
1236 return Register();
1237}
1238
1239Register fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, Register Op0) {
1240 if (RetVT.SimpleTy != MVT::v2i32)
1241 return Register();
1242 if ((Subtarget->isNeonAvailable())) {
1243 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv2i32, RC: &AArch64::FPR64RegClass, Op0);
1244 }
1245 return Register();
1246}
1247
1248Register fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, Register Op0) {
1249 if (RetVT.SimpleTy != MVT::v4i32)
1250 return Register();
1251 if ((Subtarget->isNeonAvailable())) {
1252 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv4i32, RC: &AArch64::FPR128RegClass, Op0);
1253 }
1254 return Register();
1255}
1256
1257Register fastEmit_ISD_ABS_MVT_v1i64_r(MVT RetVT, Register Op0) {
1258 if (RetVT.SimpleTy != MVT::v1i64)
1259 return Register();
1260 if ((Subtarget->isNeonAvailable())) {
1261 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
1262 }
1263 return Register();
1264}
1265
1266Register fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, Register Op0) {
1267 if (RetVT.SimpleTy != MVT::v2i64)
1268 return Register();
1269 if ((Subtarget->isNeonAvailable())) {
1270 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv2i64, RC: &AArch64::FPR128RegClass, Op0);
1271 }
1272 return Register();
1273}
1274
1275Register fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, Register Op0) {
1276 switch (VT.SimpleTy) {
1277 case MVT::i32: return fastEmit_ISD_ABS_MVT_i32_r(RetVT, Op0);
1278 case MVT::i64: return fastEmit_ISD_ABS_MVT_i64_r(RetVT, Op0);
1279 case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0);
1280 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
1281 case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0);
1282 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
1283 case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0);
1284 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
1285 case MVT::v1i64: return fastEmit_ISD_ABS_MVT_v1i64_r(RetVT, Op0);
1286 case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0);
1287 default: return Register();
1288 }
1289}
1290
1291// FastEmit functions for ISD::BITCAST.
1292
1293Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Register Op0) {
1294 if ((!Subtarget->isLittleEndian())) {
1295 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1296 }
1297 return Register();
1298}
1299
1300Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Register Op0) {
1301 if ((!Subtarget->isLittleEndian())) {
1302 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1303 }
1304 return Register();
1305}
1306
1307Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Register Op0) {
1308 if ((!Subtarget->isLittleEndian())) {
1309 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1310 }
1311 return Register();
1312}
1313
1314Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Register Op0) {
1315 if ((!Subtarget->isLittleEndian())) {
1316 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1317 }
1318 return Register();
1319}
1320
1321Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Register Op0) {
1322 if ((!Subtarget->isLittleEndian())) {
1323 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1324 }
1325 return Register();
1326}
1327
1328Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Register Op0) {
1329 if ((!Subtarget->isLittleEndian())) {
1330 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1331 }
1332 return Register();
1333}
1334
1335Register fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, Register Op0) {
1336switch (RetVT.SimpleTy) {
1337 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0);
1338 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0);
1339 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0);
1340 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0);
1341 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Op0);
1342 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0);
1343 default: return Register();
1344}
1345}
1346
1347Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Register Op0) {
1348 if ((!Subtarget->isLittleEndian())) {
1349 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1350 }
1351 return Register();
1352}
1353
1354Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Register Op0) {
1355 if ((!Subtarget->isLittleEndian())) {
1356 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1357 }
1358 return Register();
1359}
1360
1361Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Register Op0) {
1362 if ((!Subtarget->isLittleEndian())) {
1363 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1364 }
1365 return Register();
1366}
1367
1368Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Register Op0) {
1369 if ((!Subtarget->isLittleEndian())) {
1370 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1371 }
1372 return Register();
1373}
1374
1375Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Register Op0) {
1376 if ((!Subtarget->isLittleEndian())) {
1377 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1378 }
1379 return Register();
1380}
1381
1382Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Register Op0) {
1383 if ((!Subtarget->isLittleEndian())) {
1384 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1385 }
1386 return Register();
1387}
1388
1389Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Register Op0) {
1390 if ((!Subtarget->isLittleEndian())) {
1391 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1392 }
1393 return Register();
1394}
1395
1396Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Register Op0) {
1397 if ((!Subtarget->isLittleEndian())) {
1398 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1399 }
1400 return Register();
1401}
1402
1403Register fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, Register Op0) {
1404switch (RetVT.SimpleTy) {
1405 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0);
1406 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0);
1407 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0);
1408 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0);
1409 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0);
1410 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Op0);
1411 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0);
1412 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Op0);
1413 default: return Register();
1414}
1415}
1416
1417Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Register Op0) {
1418 if ((!Subtarget->isLittleEndian())) {
1419 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1420 }
1421 return Register();
1422}
1423
1424Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Register Op0) {
1425 if ((!Subtarget->isLittleEndian())) {
1426 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1427 }
1428 return Register();
1429}
1430
1431Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Register Op0) {
1432 if ((!Subtarget->isLittleEndian())) {
1433 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1434 }
1435 return Register();
1436}
1437
1438Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Register Op0) {
1439 if ((!Subtarget->isLittleEndian())) {
1440 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1441 }
1442 return Register();
1443}
1444
1445Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Register Op0) {
1446 if ((!Subtarget->isLittleEndian())) {
1447 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1448 }
1449 return Register();
1450}
1451
1452Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Register Op0) {
1453 if ((!Subtarget->isLittleEndian())) {
1454 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1455 }
1456 return Register();
1457}
1458
1459Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Register Op0) {
1460 if ((!Subtarget->isLittleEndian())) {
1461 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1462 }
1463 return Register();
1464}
1465
1466Register fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, Register Op0) {
1467switch (RetVT.SimpleTy) {
1468 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0);
1469 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0);
1470 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0);
1471 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0);
1472 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Op0);
1473 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0);
1474 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0);
1475 default: return Register();
1476}
1477}
1478
1479Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Register Op0) {
1480 if ((!Subtarget->isLittleEndian())) {
1481 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1482 }
1483 return Register();
1484}
1485
1486Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Register Op0) {
1487 if ((!Subtarget->isLittleEndian())) {
1488 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1489 }
1490 return Register();
1491}
1492
1493Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Register Op0) {
1494 if ((!Subtarget->isLittleEndian())) {
1495 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1496 }
1497 return Register();
1498}
1499
1500Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Register Op0) {
1501 if ((!Subtarget->isLittleEndian())) {
1502 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1503 }
1504 return Register();
1505}
1506
1507Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Register Op0) {
1508 if ((!Subtarget->isLittleEndian())) {
1509 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1510 }
1511 return Register();
1512}
1513
1514Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Register Op0) {
1515 if ((!Subtarget->isLittleEndian())) {
1516 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1517 }
1518 return Register();
1519}
1520
1521Register fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, Register Op0) {
1522switch (RetVT.SimpleTy) {
1523 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0);
1524 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0);
1525 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0);
1526 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0);
1527 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0);
1528 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Op0);
1529 default: return Register();
1530}
1531}
1532
1533Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Register Op0) {
1534 if ((!Subtarget->isLittleEndian())) {
1535 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1536 }
1537 return Register();
1538}
1539
1540Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Register Op0) {
1541 if ((!Subtarget->isLittleEndian())) {
1542 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1543 }
1544 return Register();
1545}
1546
1547Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Register Op0) {
1548 if ((!Subtarget->isLittleEndian())) {
1549 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1550 }
1551 return Register();
1552}
1553
1554Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Register Op0) {
1555 if ((!Subtarget->isLittleEndian())) {
1556 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1557 }
1558 return Register();
1559}
1560
1561Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Register Op0) {
1562 if ((!Subtarget->isLittleEndian())) {
1563 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1564 }
1565 return Register();
1566}
1567
1568Register fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, Register Op0) {
1569switch (RetVT.SimpleTy) {
1570 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0);
1571 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0);
1572 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0);
1573 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0);
1574 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0);
1575 default: return Register();
1576}
1577}
1578
1579Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Register Op0) {
1580 if ((!Subtarget->isLittleEndian())) {
1581 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1582 }
1583 return Register();
1584}
1585
1586Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Register Op0) {
1587 if ((!Subtarget->isLittleEndian())) {
1588 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1589 }
1590 return Register();
1591}
1592
1593Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Register Op0) {
1594 if ((!Subtarget->isLittleEndian())) {
1595 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1596 }
1597 return Register();
1598}
1599
1600Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Register Op0) {
1601 if ((!Subtarget->isLittleEndian())) {
1602 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1603 }
1604 return Register();
1605}
1606
1607Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Register Op0) {
1608 if ((!Subtarget->isLittleEndian())) {
1609 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1610 }
1611 return Register();
1612}
1613
1614Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Register Op0) {
1615 if ((!Subtarget->isLittleEndian())) {
1616 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1617 }
1618 return Register();
1619}
1620
1621Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Register Op0) {
1622 if ((!Subtarget->isLittleEndian())) {
1623 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1624 }
1625 return Register();
1626}
1627
1628Register fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, Register Op0) {
1629switch (RetVT.SimpleTy) {
1630 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0);
1631 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0);
1632 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0);
1633 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0);
1634 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0);
1635 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Op0);
1636 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Op0);
1637 default: return Register();
1638}
1639}
1640
1641Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Register Op0) {
1642 if ((!Subtarget->isLittleEndian())) {
1643 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1644 }
1645 return Register();
1646}
1647
1648Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Register Op0) {
1649 if ((!Subtarget->isLittleEndian())) {
1650 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1651 }
1652 return Register();
1653}
1654
1655Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Register Op0) {
1656 if ((!Subtarget->isLittleEndian())) {
1657 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1658 }
1659 return Register();
1660}
1661
1662Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Register Op0) {
1663 if ((!Subtarget->isLittleEndian())) {
1664 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1665 }
1666 return Register();
1667}
1668
1669Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Register Op0) {
1670 if ((!Subtarget->isLittleEndian())) {
1671 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1672 }
1673 return Register();
1674}
1675
1676Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Register Op0) {
1677 if ((!Subtarget->isLittleEndian())) {
1678 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1679 }
1680 return Register();
1681}
1682
1683Register fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, Register Op0) {
1684switch (RetVT.SimpleTy) {
1685 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0);
1686 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0);
1687 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0);
1688 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0);
1689 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Op0);
1690 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0);
1691 default: return Register();
1692}
1693}
1694
1695Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Register Op0) {
1696 if ((!Subtarget->isLittleEndian())) {
1697 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1698 }
1699 return Register();
1700}
1701
1702Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Register Op0) {
1703 if ((!Subtarget->isLittleEndian())) {
1704 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1705 }
1706 return Register();
1707}
1708
1709Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Register Op0) {
1710 if ((!Subtarget->isLittleEndian())) {
1711 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1712 }
1713 return Register();
1714}
1715
1716Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Register Op0) {
1717 if ((!Subtarget->isLittleEndian())) {
1718 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1719 }
1720 return Register();
1721}
1722
1723Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Register Op0) {
1724 if ((!Subtarget->isLittleEndian())) {
1725 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1726 }
1727 return Register();
1728}
1729
1730Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Register Op0) {
1731 if ((!Subtarget->isLittleEndian())) {
1732 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1733 }
1734 return Register();
1735}
1736
1737Register fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, Register Op0) {
1738switch (RetVT.SimpleTy) {
1739 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0);
1740 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0);
1741 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0);
1742 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0);
1743 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Op0);
1744 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0);
1745 default: return Register();
1746}
1747}
1748
1749Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Register Op0) {
1750 if ((!Subtarget->isLittleEndian())) {
1751 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1752 }
1753 return Register();
1754}
1755
1756Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Register Op0) {
1757 if ((!Subtarget->isLittleEndian())) {
1758 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1759 }
1760 return Register();
1761}
1762
1763Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Register Op0) {
1764 if ((!Subtarget->isLittleEndian())) {
1765 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1766 }
1767 return Register();
1768}
1769
1770Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Register Op0) {
1771 if ((!Subtarget->isLittleEndian())) {
1772 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1773 }
1774 return Register();
1775}
1776
1777Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Register Op0) {
1778 if ((!Subtarget->isLittleEndian())) {
1779 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1780 }
1781 return Register();
1782}
1783
1784Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Register Op0) {
1785 if ((!Subtarget->isLittleEndian())) {
1786 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1787 }
1788 return Register();
1789}
1790
1791Register fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, Register Op0) {
1792switch (RetVT.SimpleTy) {
1793 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0);
1794 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0);
1795 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0);
1796 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0);
1797 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Op0);
1798 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0);
1799 default: return Register();
1800}
1801}
1802
1803Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Register Op0) {
1804 if ((!Subtarget->isLittleEndian())) {
1805 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1806 }
1807 return Register();
1808}
1809
1810Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Register Op0) {
1811 if ((!Subtarget->isLittleEndian())) {
1812 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1813 }
1814 return Register();
1815}
1816
1817Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Register Op0) {
1818 if ((!Subtarget->isLittleEndian())) {
1819 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1820 }
1821 return Register();
1822}
1823
1824Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Register Op0) {
1825 if ((!Subtarget->isLittleEndian())) {
1826 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1827 }
1828 return Register();
1829}
1830
1831Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Register Op0) {
1832 if ((!Subtarget->isLittleEndian())) {
1833 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1834 }
1835 return Register();
1836}
1837
1838Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Register Op0) {
1839 if ((!Subtarget->isLittleEndian())) {
1840 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1841 }
1842 return Register();
1843}
1844
1845Register fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, Register Op0) {
1846switch (RetVT.SimpleTy) {
1847 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0);
1848 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0);
1849 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0);
1850 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0);
1851 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0);
1852 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Op0);
1853 default: return Register();
1854}
1855}
1856
1857Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Register Op0) {
1858 if ((!Subtarget->isLittleEndian())) {
1859 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1860 }
1861 return Register();
1862}
1863
1864Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Register Op0) {
1865 if ((!Subtarget->isLittleEndian())) {
1866 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1867 }
1868 return Register();
1869}
1870
1871Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Register Op0) {
1872 if ((!Subtarget->isLittleEndian())) {
1873 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1874 }
1875 return Register();
1876}
1877
1878Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Register Op0) {
1879 if ((!Subtarget->isLittleEndian())) {
1880 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1881 }
1882 return Register();
1883}
1884
1885Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Register Op0) {
1886 if ((!Subtarget->isLittleEndian())) {
1887 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1888 }
1889 return Register();
1890}
1891
1892Register fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, Register Op0) {
1893switch (RetVT.SimpleTy) {
1894 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0);
1895 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0);
1896 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0);
1897 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0);
1898 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0);
1899 default: return Register();
1900}
1901}
1902
1903Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Register Op0) {
1904 if ((!Subtarget->isLittleEndian())) {
1905 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1906 }
1907 return Register();
1908}
1909
1910Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Register Op0) {
1911 if ((!Subtarget->isLittleEndian())) {
1912 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1913 }
1914 return Register();
1915}
1916
1917Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Register Op0) {
1918 if ((!Subtarget->isLittleEndian())) {
1919 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1920 }
1921 return Register();
1922}
1923
1924Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Register Op0) {
1925 if ((!Subtarget->isLittleEndian())) {
1926 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1927 }
1928 return Register();
1929}
1930
1931Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Register Op0) {
1932 if ((!Subtarget->isLittleEndian())) {
1933 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1934 }
1935 return Register();
1936}
1937
1938Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(Register Op0) {
1939 if ((!Subtarget->isLittleEndian())) {
1940 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1941 }
1942 return Register();
1943}
1944
1945Register fastEmit_ISD_BITCAST_MVT_v4bf16_r(MVT RetVT, Register Op0) {
1946switch (RetVT.SimpleTy) {
1947 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Op0);
1948 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Op0);
1949 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Op0);
1950 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Op0);
1951 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Op0);
1952 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(Op0);
1953 default: return Register();
1954}
1955}
1956
1957Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Register Op0) {
1958 if ((!Subtarget->isLittleEndian())) {
1959 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1960 }
1961 return Register();
1962}
1963
1964Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Register Op0) {
1965 if ((!Subtarget->isLittleEndian())) {
1966 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1967 }
1968 return Register();
1969}
1970
1971Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Register Op0) {
1972 if ((!Subtarget->isLittleEndian())) {
1973 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1974 }
1975 return Register();
1976}
1977
1978Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Register Op0) {
1979 if ((!Subtarget->isLittleEndian())) {
1980 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1981 }
1982 return Register();
1983}
1984
1985Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Register Op0) {
1986 if ((!Subtarget->isLittleEndian())) {
1987 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1988 }
1989 return Register();
1990}
1991
1992Register fastEmit_ISD_BITCAST_MVT_v8bf16_r(MVT RetVT, Register Op0) {
1993switch (RetVT.SimpleTy) {
1994 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Op0);
1995 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Op0);
1996 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Op0);
1997 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Op0);
1998 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Op0);
1999 default: return Register();
2000}
2001}
2002
2003Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Register Op0) {
2004 if ((!Subtarget->isLittleEndian())) {
2005 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2006 }
2007 return Register();
2008}
2009
2010Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Register Op0) {
2011 if ((!Subtarget->isLittleEndian())) {
2012 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
2013 }
2014 return Register();
2015}
2016
2017Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Register Op0) {
2018 if ((!Subtarget->isLittleEndian())) {
2019 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2020 }
2021 return Register();
2022}
2023
2024Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Register Op0) {
2025 if ((!Subtarget->isLittleEndian())) {
2026 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2027 }
2028 return Register();
2029}
2030
2031Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Register Op0) {
2032 if ((!Subtarget->isLittleEndian())) {
2033 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2034 }
2035 return Register();
2036}
2037
2038Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Register Op0) {
2039 if ((!Subtarget->isLittleEndian())) {
2040 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2041 }
2042 return Register();
2043}
2044
2045Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Register Op0) {
2046 if ((!Subtarget->isLittleEndian())) {
2047 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2048 }
2049 return Register();
2050}
2051
2052Register fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, Register Op0) {
2053switch (RetVT.SimpleTy) {
2054 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0);
2055 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0);
2056 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0);
2057 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0);
2058 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0);
2059 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Op0);
2060 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Op0);
2061 default: return Register();
2062}
2063}
2064
2065Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Register Op0) {
2066 if ((!Subtarget->isLittleEndian())) {
2067 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
2068 }
2069 return Register();
2070}
2071
2072Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Register Op0) {
2073 if ((!Subtarget->isLittleEndian())) {
2074 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2075 }
2076 return Register();
2077}
2078
2079Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Register Op0) {
2080 if ((!Subtarget->isLittleEndian())) {
2081 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2082 }
2083 return Register();
2084}
2085
2086Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Register Op0) {
2087 if ((!Subtarget->isLittleEndian())) {
2088 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2089 }
2090 return Register();
2091}
2092
2093Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Register Op0) {
2094 if ((!Subtarget->isLittleEndian())) {
2095 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2096 }
2097 return Register();
2098}
2099
2100Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Register Op0) {
2101 if ((!Subtarget->isLittleEndian())) {
2102 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2103 }
2104 return Register();
2105}
2106
2107Register fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, Register Op0) {
2108switch (RetVT.SimpleTy) {
2109 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0);
2110 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0);
2111 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0);
2112 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0);
2113 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Op0);
2114 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0);
2115 default: return Register();
2116}
2117}
2118
2119Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Register Op0) {
2120 if ((!Subtarget->isLittleEndian())) {
2121 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
2122 }
2123 return Register();
2124}
2125
2126Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Register Op0) {
2127 if ((!Subtarget->isLittleEndian())) {
2128 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2129 }
2130 return Register();
2131}
2132
2133Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Register Op0) {
2134 if ((!Subtarget->isLittleEndian())) {
2135 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2136 }
2137 return Register();
2138}
2139
2140Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Register Op0) {
2141 if ((!Subtarget->isLittleEndian())) {
2142 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2143 }
2144 return Register();
2145}
2146
2147Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(Register Op0) {
2148 if ((!Subtarget->isLittleEndian())) {
2149 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2150 }
2151 return Register();
2152}
2153
2154Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Register Op0) {
2155 if ((!Subtarget->isLittleEndian())) {
2156 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2157 }
2158 return Register();
2159}
2160
2161Register fastEmit_ISD_BITCAST_MVT_v1f64_r(MVT RetVT, Register Op0) {
2162switch (RetVT.SimpleTy) {
2163 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Op0);
2164 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Op0);
2165 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Op0);
2166 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Op0);
2167 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(Op0);
2168 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Op0);
2169 default: return Register();
2170}
2171}
2172
2173Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Register Op0) {
2174 if ((!Subtarget->isLittleEndian())) {
2175 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2176 }
2177 return Register();
2178}
2179
2180Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Register Op0) {
2181 if ((!Subtarget->isLittleEndian())) {
2182 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2183 }
2184 return Register();
2185}
2186
2187Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Register Op0) {
2188 if ((!Subtarget->isLittleEndian())) {
2189 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2190 }
2191 return Register();
2192}
2193
2194Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Register Op0) {
2195 if ((!Subtarget->isLittleEndian())) {
2196 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2197 }
2198 return Register();
2199}
2200
2201Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Register Op0) {
2202 if ((!Subtarget->isLittleEndian())) {
2203 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2204 }
2205 return Register();
2206}
2207
2208Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Register Op0) {
2209 if ((!Subtarget->isLittleEndian())) {
2210 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2211 }
2212 return Register();
2213}
2214
2215Register fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, Register Op0) {
2216switch (RetVT.SimpleTy) {
2217 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0);
2218 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0);
2219 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0);
2220 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0);
2221 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Op0);
2222 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0);
2223 default: return Register();
2224}
2225}
2226
2227Register fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, Register Op0) {
2228 switch (VT.SimpleTy) {
2229 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
2230 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0);
2231 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0);
2232 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0);
2233 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0);
2234 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0);
2235 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0);
2236 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0);
2237 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0);
2238 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0);
2239 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0);
2240 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v4bf16_r(RetVT, Op0);
2241 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v8bf16_r(RetVT, Op0);
2242 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0);
2243 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0);
2244 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v1f64_r(RetVT, Op0);
2245 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0);
2246 default: return Register();
2247 }
2248}
2249
2250// FastEmit functions for ISD::BITREVERSE.
2251
2252Register fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, Register Op0) {
2253 if (RetVT.SimpleTy != MVT::i32)
2254 return Register();
2255 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITWr, RC: &AArch64::GPR32RegClass, Op0);
2256}
2257
2258Register fastEmit_ISD_BITREVERSE_MVT_i64_r(MVT RetVT, Register Op0) {
2259 if (RetVT.SimpleTy != MVT::i64)
2260 return Register();
2261 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITXr, RC: &AArch64::GPR64RegClass, Op0);
2262}
2263
2264Register fastEmit_ISD_BITREVERSE_MVT_v8i8_r(MVT RetVT, Register Op0) {
2265 if (RetVT.SimpleTy != MVT::v8i8)
2266 return Register();
2267 if ((Subtarget->isNeonAvailable())) {
2268 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITv8i8, RC: &AArch64::FPR64RegClass, Op0);
2269 }
2270 return Register();
2271}
2272
2273Register fastEmit_ISD_BITREVERSE_MVT_v16i8_r(MVT RetVT, Register Op0) {
2274 if (RetVT.SimpleTy != MVT::v16i8)
2275 return Register();
2276 if ((Subtarget->isNeonAvailable())) {
2277 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITv16i8, RC: &AArch64::FPR128RegClass, Op0);
2278 }
2279 return Register();
2280}
2281
2282Register fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, Register Op0) {
2283 switch (VT.SimpleTy) {
2284 case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0);
2285 case MVT::i64: return fastEmit_ISD_BITREVERSE_MVT_i64_r(RetVT, Op0);
2286 case MVT::v8i8: return fastEmit_ISD_BITREVERSE_MVT_v8i8_r(RetVT, Op0);
2287 case MVT::v16i8: return fastEmit_ISD_BITREVERSE_MVT_v16i8_r(RetVT, Op0);
2288 default: return Register();
2289 }
2290}
2291
2292// FastEmit functions for ISD::BSWAP.
2293
2294Register fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, Register Op0) {
2295 if (RetVT.SimpleTy != MVT::i32)
2296 return Register();
2297 return fastEmitInst_r(MachineInstOpcode: AArch64::REVWr, RC: &AArch64::GPR32RegClass, Op0);
2298}
2299
2300Register fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, Register Op0) {
2301 if (RetVT.SimpleTy != MVT::i64)
2302 return Register();
2303 return fastEmitInst_r(MachineInstOpcode: AArch64::REVXr, RC: &AArch64::GPR64RegClass, Op0);
2304}
2305
2306Register fastEmit_ISD_BSWAP_MVT_v4i16_r(MVT RetVT, Register Op0) {
2307 if (RetVT.SimpleTy != MVT::v4i16)
2308 return Register();
2309 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
2310}
2311
2312Register fastEmit_ISD_BSWAP_MVT_v8i16_r(MVT RetVT, Register Op0) {
2313 if (RetVT.SimpleTy != MVT::v8i16)
2314 return Register();
2315 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
2316}
2317
2318Register fastEmit_ISD_BSWAP_MVT_v2i32_r(MVT RetVT, Register Op0) {
2319 if (RetVT.SimpleTy != MVT::v2i32)
2320 return Register();
2321 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
2322}
2323
2324Register fastEmit_ISD_BSWAP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2325 if (RetVT.SimpleTy != MVT::v4i32)
2326 return Register();
2327 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
2328}
2329
2330Register fastEmit_ISD_BSWAP_MVT_v2i64_r(MVT RetVT, Register Op0) {
2331 if (RetVT.SimpleTy != MVT::v2i64)
2332 return Register();
2333 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2334}
2335
2336Register fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, Register Op0) {
2337 switch (VT.SimpleTy) {
2338 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
2339 case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0);
2340 case MVT::v4i16: return fastEmit_ISD_BSWAP_MVT_v4i16_r(RetVT, Op0);
2341 case MVT::v8i16: return fastEmit_ISD_BSWAP_MVT_v8i16_r(RetVT, Op0);
2342 case MVT::v2i32: return fastEmit_ISD_BSWAP_MVT_v2i32_r(RetVT, Op0);
2343 case MVT::v4i32: return fastEmit_ISD_BSWAP_MVT_v4i32_r(RetVT, Op0);
2344 case MVT::v2i64: return fastEmit_ISD_BSWAP_MVT_v2i64_r(RetVT, Op0);
2345 default: return Register();
2346 }
2347}
2348
2349// FastEmit functions for ISD::CTLS.
2350
2351Register fastEmit_ISD_CTLS_MVT_i32_r(MVT RetVT, Register Op0) {
2352 if (RetVT.SimpleTy != MVT::i32)
2353 return Register();
2354 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSWr, RC: &AArch64::GPR32RegClass, Op0);
2355}
2356
2357Register fastEmit_ISD_CTLS_MVT_i64_r(MVT RetVT, Register Op0) {
2358 if (RetVT.SimpleTy != MVT::i64)
2359 return Register();
2360 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSXr, RC: &AArch64::GPR64RegClass, Op0);
2361}
2362
2363Register fastEmit_ISD_CTLS_MVT_v8i8_r(MVT RetVT, Register Op0) {
2364 if (RetVT.SimpleTy != MVT::v8i8)
2365 return Register();
2366 if ((Subtarget->isNeonAvailable())) {
2367 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv8i8, RC: &AArch64::FPR64RegClass, Op0);
2368 }
2369 return Register();
2370}
2371
2372Register fastEmit_ISD_CTLS_MVT_v16i8_r(MVT RetVT, Register Op0) {
2373 if (RetVT.SimpleTy != MVT::v16i8)
2374 return Register();
2375 if ((Subtarget->isNeonAvailable())) {
2376 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv16i8, RC: &AArch64::FPR128RegClass, Op0);
2377 }
2378 return Register();
2379}
2380
2381Register fastEmit_ISD_CTLS_MVT_v4i16_r(MVT RetVT, Register Op0) {
2382 if (RetVT.SimpleTy != MVT::v4i16)
2383 return Register();
2384 if ((Subtarget->isNeonAvailable())) {
2385 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv4i16, RC: &AArch64::FPR64RegClass, Op0);
2386 }
2387 return Register();
2388}
2389
2390Register fastEmit_ISD_CTLS_MVT_v8i16_r(MVT RetVT, Register Op0) {
2391 if (RetVT.SimpleTy != MVT::v8i16)
2392 return Register();
2393 if ((Subtarget->isNeonAvailable())) {
2394 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv8i16, RC: &AArch64::FPR128RegClass, Op0);
2395 }
2396 return Register();
2397}
2398
2399Register fastEmit_ISD_CTLS_MVT_v2i32_r(MVT RetVT, Register Op0) {
2400 if (RetVT.SimpleTy != MVT::v2i32)
2401 return Register();
2402 if ((Subtarget->isNeonAvailable())) {
2403 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv2i32, RC: &AArch64::FPR64RegClass, Op0);
2404 }
2405 return Register();
2406}
2407
2408Register fastEmit_ISD_CTLS_MVT_v4i32_r(MVT RetVT, Register Op0) {
2409 if (RetVT.SimpleTy != MVT::v4i32)
2410 return Register();
2411 if ((Subtarget->isNeonAvailable())) {
2412 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv4i32, RC: &AArch64::FPR128RegClass, Op0);
2413 }
2414 return Register();
2415}
2416
2417Register fastEmit_ISD_CTLS_r(MVT VT, MVT RetVT, Register Op0) {
2418 switch (VT.SimpleTy) {
2419 case MVT::i32: return fastEmit_ISD_CTLS_MVT_i32_r(RetVT, Op0);
2420 case MVT::i64: return fastEmit_ISD_CTLS_MVT_i64_r(RetVT, Op0);
2421 case MVT::v8i8: return fastEmit_ISD_CTLS_MVT_v8i8_r(RetVT, Op0);
2422 case MVT::v16i8: return fastEmit_ISD_CTLS_MVT_v16i8_r(RetVT, Op0);
2423 case MVT::v4i16: return fastEmit_ISD_CTLS_MVT_v4i16_r(RetVT, Op0);
2424 case MVT::v8i16: return fastEmit_ISD_CTLS_MVT_v8i16_r(RetVT, Op0);
2425 case MVT::v2i32: return fastEmit_ISD_CTLS_MVT_v2i32_r(RetVT, Op0);
2426 case MVT::v4i32: return fastEmit_ISD_CTLS_MVT_v4i32_r(RetVT, Op0);
2427 default: return Register();
2428 }
2429}
2430
2431// FastEmit functions for ISD::CTLZ.
2432
2433Register fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, Register Op0) {
2434 if (RetVT.SimpleTy != MVT::i32)
2435 return Register();
2436 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZWr, RC: &AArch64::GPR32RegClass, Op0);
2437}
2438
2439Register fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, Register Op0) {
2440 if (RetVT.SimpleTy != MVT::i64)
2441 return Register();
2442 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZXr, RC: &AArch64::GPR64RegClass, Op0);
2443}
2444
2445Register fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, Register Op0) {
2446 if (RetVT.SimpleTy != MVT::v8i8)
2447 return Register();
2448 if ((Subtarget->isNeonAvailable())) {
2449 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv8i8, RC: &AArch64::FPR64RegClass, Op0);
2450 }
2451 return Register();
2452}
2453
2454Register fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, Register Op0) {
2455 if (RetVT.SimpleTy != MVT::v16i8)
2456 return Register();
2457 if ((Subtarget->isNeonAvailable())) {
2458 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv16i8, RC: &AArch64::FPR128RegClass, Op0);
2459 }
2460 return Register();
2461}
2462
2463Register fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, Register Op0) {
2464 if (RetVT.SimpleTy != MVT::v4i16)
2465 return Register();
2466 if ((Subtarget->isNeonAvailable())) {
2467 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv4i16, RC: &AArch64::FPR64RegClass, Op0);
2468 }
2469 return Register();
2470}
2471
2472Register fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, Register Op0) {
2473 if (RetVT.SimpleTy != MVT::v8i16)
2474 return Register();
2475 if ((Subtarget->isNeonAvailable())) {
2476 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv8i16, RC: &AArch64::FPR128RegClass, Op0);
2477 }
2478 return Register();
2479}
2480
2481Register fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, Register Op0) {
2482 if (RetVT.SimpleTy != MVT::v2i32)
2483 return Register();
2484 if ((Subtarget->isNeonAvailable())) {
2485 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv2i32, RC: &AArch64::FPR64RegClass, Op0);
2486 }
2487 return Register();
2488}
2489
2490Register fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, Register Op0) {
2491 if (RetVT.SimpleTy != MVT::v4i32)
2492 return Register();
2493 if ((Subtarget->isNeonAvailable())) {
2494 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv4i32, RC: &AArch64::FPR128RegClass, Op0);
2495 }
2496 return Register();
2497}
2498
2499Register fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, Register Op0) {
2500 switch (VT.SimpleTy) {
2501 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
2502 case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0);
2503 case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0);
2504 case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0);
2505 case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0);
2506 case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0);
2507 case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0);
2508 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
2509 default: return Register();
2510 }
2511}
2512
2513// FastEmit functions for ISD::CTPOP.
2514
2515Register fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, Register Op0) {
2516 if (RetVT.SimpleTy != MVT::i32)
2517 return Register();
2518 if ((Subtarget->hasCSSC())) {
2519 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTWr, RC: &AArch64::GPR32RegClass, Op0);
2520 }
2521 return Register();
2522}
2523
2524Register fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, Register Op0) {
2525 if (RetVT.SimpleTy != MVT::i64)
2526 return Register();
2527 if ((Subtarget->hasCSSC())) {
2528 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTXr, RC: &AArch64::GPR64RegClass, Op0);
2529 }
2530 return Register();
2531}
2532
2533Register fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, Register Op0) {
2534 if (RetVT.SimpleTy != MVT::v8i8)
2535 return Register();
2536 if ((Subtarget->isNeonAvailable())) {
2537 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTv8i8, RC: &AArch64::FPR64RegClass, Op0);
2538 }
2539 return Register();
2540}
2541
2542Register fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, Register Op0) {
2543 if (RetVT.SimpleTy != MVT::v16i8)
2544 return Register();
2545 if ((Subtarget->isNeonAvailable())) {
2546 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTv16i8, RC: &AArch64::FPR128RegClass, Op0);
2547 }
2548 return Register();
2549}
2550
2551Register fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, Register Op0) {
2552 switch (VT.SimpleTy) {
2553 case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0);
2554 case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0);
2555 case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0);
2556 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
2557 default: return Register();
2558 }
2559}
2560
2561// FastEmit functions for ISD::CTTZ.
2562
2563Register fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, Register Op0) {
2564 if (RetVT.SimpleTy != MVT::i32)
2565 return Register();
2566 if ((Subtarget->hasCSSC())) {
2567 return fastEmitInst_r(MachineInstOpcode: AArch64::CTZWr, RC: &AArch64::GPR32RegClass, Op0);
2568 }
2569 return Register();
2570}
2571
2572Register fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, Register Op0) {
2573 if (RetVT.SimpleTy != MVT::i64)
2574 return Register();
2575 if ((Subtarget->hasCSSC())) {
2576 return fastEmitInst_r(MachineInstOpcode: AArch64::CTZXr, RC: &AArch64::GPR64RegClass, Op0);
2577 }
2578 return Register();
2579}
2580
2581Register fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, Register Op0) {
2582 switch (VT.SimpleTy) {
2583 case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0);
2584 case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0);
2585 default: return Register();
2586 }
2587}
2588
2589// FastEmit functions for ISD::FABS.
2590
2591Register fastEmit_ISD_FABS_MVT_bf16_r(MVT RetVT, Register Op0) {
2592 if (RetVT.SimpleTy != MVT::bf16)
2593 return Register();
2594 if ((Subtarget->hasFullFP16())) {
2595 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSHr, RC: &AArch64::FPR16RegClass, Op0);
2596 }
2597 return Register();
2598}
2599
2600Register fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, Register Op0) {
2601 if (RetVT.SimpleTy != MVT::f16)
2602 return Register();
2603 if ((Subtarget->hasFullFP16())) {
2604 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSHr, RC: &AArch64::FPR16RegClass, Op0);
2605 }
2606 return Register();
2607}
2608
2609Register fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, Register Op0) {
2610 if (RetVT.SimpleTy != MVT::f32)
2611 return Register();
2612 if ((Subtarget->hasFPARMv8())) {
2613 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSSr, RC: &AArch64::FPR32RegClass, Op0);
2614 }
2615 return Register();
2616}
2617
2618Register fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, Register Op0) {
2619 if (RetVT.SimpleTy != MVT::f64)
2620 return Register();
2621 if ((Subtarget->hasFPARMv8())) {
2622 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSDr, RC: &AArch64::FPR64RegClass, Op0);
2623 }
2624 return Register();
2625}
2626
2627Register fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, Register Op0) {
2628 if (RetVT.SimpleTy != MVT::v4f16)
2629 return Register();
2630 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2631 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f16, RC: &AArch64::FPR64RegClass, Op0);
2632 }
2633 return Register();
2634}
2635
2636Register fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, Register Op0) {
2637 if (RetVT.SimpleTy != MVT::v8f16)
2638 return Register();
2639 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2640 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv8f16, RC: &AArch64::FPR128RegClass, Op0);
2641 }
2642 return Register();
2643}
2644
2645Register fastEmit_ISD_FABS_MVT_v4bf16_r(MVT RetVT, Register Op0) {
2646 if (RetVT.SimpleTy != MVT::v4bf16)
2647 return Register();
2648 if ((Subtarget->hasFullFP16())) {
2649 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f16, RC: &AArch64::FPR64RegClass, Op0);
2650 }
2651 return Register();
2652}
2653
2654Register fastEmit_ISD_FABS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
2655 if (RetVT.SimpleTy != MVT::v8bf16)
2656 return Register();
2657 if ((Subtarget->hasFullFP16())) {
2658 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv8f16, RC: &AArch64::FPR128RegClass, Op0);
2659 }
2660 return Register();
2661}
2662
2663Register fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, Register Op0) {
2664 if (RetVT.SimpleTy != MVT::v2f32)
2665 return Register();
2666 if ((Subtarget->isNeonAvailable())) {
2667 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv2f32, RC: &AArch64::FPR64RegClass, Op0);
2668 }
2669 return Register();
2670}
2671
2672Register fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, Register Op0) {
2673 if (RetVT.SimpleTy != MVT::v4f32)
2674 return Register();
2675 if ((Subtarget->isNeonAvailable())) {
2676 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f32, RC: &AArch64::FPR128RegClass, Op0);
2677 }
2678 return Register();
2679}
2680
2681Register fastEmit_ISD_FABS_MVT_v2f64_r(MVT RetVT, Register Op0) {
2682 if (RetVT.SimpleTy != MVT::v2f64)
2683 return Register();
2684 if ((Subtarget->isNeonAvailable())) {
2685 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv2f64, RC: &AArch64::FPR128RegClass, Op0);
2686 }
2687 return Register();
2688}
2689
2690Register fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, Register Op0) {
2691 switch (VT.SimpleTy) {
2692 case MVT::bf16: return fastEmit_ISD_FABS_MVT_bf16_r(RetVT, Op0);
2693 case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0);
2694 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
2695 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
2696 case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0);
2697 case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0);
2698 case MVT::v4bf16: return fastEmit_ISD_FABS_MVT_v4bf16_r(RetVT, Op0);
2699 case MVT::v8bf16: return fastEmit_ISD_FABS_MVT_v8bf16_r(RetVT, Op0);
2700 case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0);
2701 case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0);
2702 case MVT::v2f64: return fastEmit_ISD_FABS_MVT_v2f64_r(RetVT, Op0);
2703 default: return Register();
2704 }
2705}
2706
2707// FastEmit functions for ISD::FCEIL.
2708
2709Register fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
2710 if (RetVT.SimpleTy != MVT::f16)
2711 return Register();
2712 if ((Subtarget->hasFullFP16())) {
2713 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPHr, RC: &AArch64::FPR16RegClass, Op0);
2714 }
2715 return Register();
2716}
2717
2718Register fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
2719 if (RetVT.SimpleTy != MVT::f32)
2720 return Register();
2721 if ((Subtarget->hasFPARMv8())) {
2722 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPSr, RC: &AArch64::FPR32RegClass, Op0);
2723 }
2724 return Register();
2725}
2726
2727Register fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
2728 if (RetVT.SimpleTy != MVT::f64)
2729 return Register();
2730 if ((Subtarget->hasFPARMv8())) {
2731 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPDr, RC: &AArch64::FPR64RegClass, Op0);
2732 }
2733 return Register();
2734}
2735
2736Register fastEmit_ISD_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
2737 if (RetVT.SimpleTy != MVT::v4f16)
2738 return Register();
2739 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2740 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f16, RC: &AArch64::FPR64RegClass, Op0);
2741 }
2742 return Register();
2743}
2744
2745Register fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
2746 if (RetVT.SimpleTy != MVT::v8f16)
2747 return Register();
2748 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2749 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv8f16, RC: &AArch64::FPR128RegClass, Op0);
2750 }
2751 return Register();
2752}
2753
2754Register fastEmit_ISD_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
2755 if (RetVT.SimpleTy != MVT::v2f32)
2756 return Register();
2757 if ((Subtarget->isNeonAvailable())) {
2758 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f32, RC: &AArch64::FPR64RegClass, Op0);
2759 }
2760 return Register();
2761}
2762
2763Register fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
2764 if (RetVT.SimpleTy != MVT::v4f32)
2765 return Register();
2766 if ((Subtarget->isNeonAvailable())) {
2767 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f32, RC: &AArch64::FPR128RegClass, Op0);
2768 }
2769 return Register();
2770}
2771
2772Register fastEmit_ISD_FCEIL_MVT_v2f64_r(MVT RetVT, Register Op0) {
2773 if (RetVT.SimpleTy != MVT::v2f64)
2774 return Register();
2775 if ((Subtarget->isNeonAvailable())) {
2776 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f64, RC: &AArch64::FPR128RegClass, Op0);
2777 }
2778 return Register();
2779}
2780
2781Register fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
2782 switch (VT.SimpleTy) {
2783 case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0);
2784 case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0);
2785 case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0);
2786 case MVT::v4f16: return fastEmit_ISD_FCEIL_MVT_v4f16_r(RetVT, Op0);
2787 case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0);
2788 case MVT::v2f32: return fastEmit_ISD_FCEIL_MVT_v2f32_r(RetVT, Op0);
2789 case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0);
2790 case MVT::v2f64: return fastEmit_ISD_FCEIL_MVT_v2f64_r(RetVT, Op0);
2791 default: return Register();
2792 }
2793}
2794
2795// FastEmit functions for ISD::FFLOOR.
2796
2797Register fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
2798 if (RetVT.SimpleTy != MVT::f16)
2799 return Register();
2800 if ((Subtarget->hasFullFP16())) {
2801 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMHr, RC: &AArch64::FPR16RegClass, Op0);
2802 }
2803 return Register();
2804}
2805
2806Register fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
2807 if (RetVT.SimpleTy != MVT::f32)
2808 return Register();
2809 if ((Subtarget->hasFPARMv8())) {
2810 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMSr, RC: &AArch64::FPR32RegClass, Op0);
2811 }
2812 return Register();
2813}
2814
2815Register fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
2816 if (RetVT.SimpleTy != MVT::f64)
2817 return Register();
2818 if ((Subtarget->hasFPARMv8())) {
2819 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMDr, RC: &AArch64::FPR64RegClass, Op0);
2820 }
2821 return Register();
2822}
2823
2824Register fastEmit_ISD_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
2825 if (RetVT.SimpleTy != MVT::v4f16)
2826 return Register();
2827 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2828 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f16, RC: &AArch64::FPR64RegClass, Op0);
2829 }
2830 return Register();
2831}
2832
2833Register fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
2834 if (RetVT.SimpleTy != MVT::v8f16)
2835 return Register();
2836 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2837 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv8f16, RC: &AArch64::FPR128RegClass, Op0);
2838 }
2839 return Register();
2840}
2841
2842Register fastEmit_ISD_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
2843 if (RetVT.SimpleTy != MVT::v2f32)
2844 return Register();
2845 if ((Subtarget->isNeonAvailable())) {
2846 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f32, RC: &AArch64::FPR64RegClass, Op0);
2847 }
2848 return Register();
2849}
2850
2851Register fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
2852 if (RetVT.SimpleTy != MVT::v4f32)
2853 return Register();
2854 if ((Subtarget->isNeonAvailable())) {
2855 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f32, RC: &AArch64::FPR128RegClass, Op0);
2856 }
2857 return Register();
2858}
2859
2860Register fastEmit_ISD_FFLOOR_MVT_v2f64_r(MVT RetVT, Register Op0) {
2861 if (RetVT.SimpleTy != MVT::v2f64)
2862 return Register();
2863 if ((Subtarget->isNeonAvailable())) {
2864 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f64, RC: &AArch64::FPR128RegClass, Op0);
2865 }
2866 return Register();
2867}
2868
2869Register fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
2870 switch (VT.SimpleTy) {
2871 case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0);
2872 case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0);
2873 case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0);
2874 case MVT::v4f16: return fastEmit_ISD_FFLOOR_MVT_v4f16_r(RetVT, Op0);
2875 case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0);
2876 case MVT::v2f32: return fastEmit_ISD_FFLOOR_MVT_v2f32_r(RetVT, Op0);
2877 case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0);
2878 case MVT::v2f64: return fastEmit_ISD_FFLOOR_MVT_v2f64_r(RetVT, Op0);
2879 default: return Register();
2880 }
2881}
2882
2883// FastEmit functions for ISD::FNEARBYINT.
2884
2885Register fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
2886 if (RetVT.SimpleTy != MVT::f16)
2887 return Register();
2888 if ((Subtarget->hasFullFP16())) {
2889 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIHr, RC: &AArch64::FPR16RegClass, Op0);
2890 }
2891 return Register();
2892}
2893
2894Register fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
2895 if (RetVT.SimpleTy != MVT::f32)
2896 return Register();
2897 if ((Subtarget->hasFPARMv8())) {
2898 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTISr, RC: &AArch64::FPR32RegClass, Op0);
2899 }
2900 return Register();
2901}
2902
2903Register fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
2904 if (RetVT.SimpleTy != MVT::f64)
2905 return Register();
2906 if ((Subtarget->hasFPARMv8())) {
2907 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIDr, RC: &AArch64::FPR64RegClass, Op0);
2908 }
2909 return Register();
2910}
2911
2912Register fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2913 if (RetVT.SimpleTy != MVT::v4f16)
2914 return Register();
2915 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2916 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f16, RC: &AArch64::FPR64RegClass, Op0);
2917 }
2918 return Register();
2919}
2920
2921Register fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2922 if (RetVT.SimpleTy != MVT::v8f16)
2923 return Register();
2924 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2925 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv8f16, RC: &AArch64::FPR128RegClass, Op0);
2926 }
2927 return Register();
2928}
2929
2930Register fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2931 if (RetVT.SimpleTy != MVT::v2f32)
2932 return Register();
2933 if ((Subtarget->isNeonAvailable())) {
2934 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f32, RC: &AArch64::FPR64RegClass, Op0);
2935 }
2936 return Register();
2937}
2938
2939Register fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2940 if (RetVT.SimpleTy != MVT::v4f32)
2941 return Register();
2942 if ((Subtarget->isNeonAvailable())) {
2943 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f32, RC: &AArch64::FPR128RegClass, Op0);
2944 }
2945 return Register();
2946}
2947
2948Register fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
2949 if (RetVT.SimpleTy != MVT::v2f64)
2950 return Register();
2951 if ((Subtarget->isNeonAvailable())) {
2952 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f64, RC: &AArch64::FPR128RegClass, Op0);
2953 }
2954 return Register();
2955}
2956
2957Register fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
2958 switch (VT.SimpleTy) {
2959 case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0);
2960 case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0);
2961 case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0);
2962 case MVT::v4f16: return fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(RetVT, Op0);
2963 case MVT::v8f16: return fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(RetVT, Op0);
2964 case MVT::v2f32: return fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(RetVT, Op0);
2965 case MVT::v4f32: return fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(RetVT, Op0);
2966 case MVT::v2f64: return fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(RetVT, Op0);
2967 default: return Register();
2968 }
2969}
2970
2971// FastEmit functions for ISD::FNEG.
2972
2973Register fastEmit_ISD_FNEG_MVT_bf16_r(MVT RetVT, Register Op0) {
2974 if (RetVT.SimpleTy != MVT::bf16)
2975 return Register();
2976 if ((Subtarget->hasFullFP16())) {
2977 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGHr, RC: &AArch64::FPR16RegClass, Op0);
2978 }
2979 return Register();
2980}
2981
2982Register fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, Register Op0) {
2983 if (RetVT.SimpleTy != MVT::f16)
2984 return Register();
2985 if ((Subtarget->hasFullFP16())) {
2986 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGHr, RC: &AArch64::FPR16RegClass, Op0);
2987 }
2988 return Register();
2989}
2990
2991Register fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, Register Op0) {
2992 if (RetVT.SimpleTy != MVT::f32)
2993 return Register();
2994 if ((Subtarget->hasFPARMv8())) {
2995 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGSr, RC: &AArch64::FPR32RegClass, Op0);
2996 }
2997 return Register();
2998}
2999
3000Register fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, Register Op0) {
3001 if (RetVT.SimpleTy != MVT::f64)
3002 return Register();
3003 if ((Subtarget->hasFPARMv8())) {
3004 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGDr, RC: &AArch64::FPR64RegClass, Op0);
3005 }
3006 return Register();
3007}
3008
3009Register fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, Register Op0) {
3010 if (RetVT.SimpleTy != MVT::v4f16)
3011 return Register();
3012 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3013 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f16, RC: &AArch64::FPR64RegClass, Op0);
3014 }
3015 return Register();
3016}
3017
3018Register fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, Register Op0) {
3019 if (RetVT.SimpleTy != MVT::v8f16)
3020 return Register();
3021 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3022 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv8f16, RC: &AArch64::FPR128RegClass, Op0);
3023 }
3024 return Register();
3025}
3026
3027Register fastEmit_ISD_FNEG_MVT_v4bf16_r(MVT RetVT, Register Op0) {
3028 if (RetVT.SimpleTy != MVT::v4bf16)
3029 return Register();
3030 if ((Subtarget->hasFullFP16())) {
3031 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f16, RC: &AArch64::FPR64RegClass, Op0);
3032 }
3033 return Register();
3034}
3035
3036Register fastEmit_ISD_FNEG_MVT_v8bf16_r(MVT RetVT, Register Op0) {
3037 if (RetVT.SimpleTy != MVT::v8bf16)
3038 return Register();
3039 if ((Subtarget->hasFullFP16())) {
3040 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv8f16, RC: &AArch64::FPR128RegClass, Op0);
3041 }
3042 return Register();
3043}
3044
3045Register fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, Register Op0) {
3046 if (RetVT.SimpleTy != MVT::v2f32)
3047 return Register();
3048 if ((Subtarget->isNeonAvailable())) {
3049 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv2f32, RC: &AArch64::FPR64RegClass, Op0);
3050 }
3051 return Register();
3052}
3053
3054Register fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, Register Op0) {
3055 if (RetVT.SimpleTy != MVT::v4f32)
3056 return Register();
3057 if ((Subtarget->isNeonAvailable())) {
3058 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f32, RC: &AArch64::FPR128RegClass, Op0);
3059 }
3060 return Register();
3061}
3062
3063Register fastEmit_ISD_FNEG_MVT_v2f64_r(MVT RetVT, Register Op0) {
3064 if (RetVT.SimpleTy != MVT::v2f64)
3065 return Register();
3066 if ((Subtarget->isNeonAvailable())) {
3067 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv2f64, RC: &AArch64::FPR128RegClass, Op0);
3068 }
3069 return Register();
3070}
3071
3072Register fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, Register Op0) {
3073 switch (VT.SimpleTy) {
3074 case MVT::bf16: return fastEmit_ISD_FNEG_MVT_bf16_r(RetVT, Op0);
3075 case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0);
3076 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
3077 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
3078 case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0);
3079 case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0);
3080 case MVT::v4bf16: return fastEmit_ISD_FNEG_MVT_v4bf16_r(RetVT, Op0);
3081 case MVT::v8bf16: return fastEmit_ISD_FNEG_MVT_v8bf16_r(RetVT, Op0);
3082 case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0);
3083 case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0);
3084 case MVT::v2f64: return fastEmit_ISD_FNEG_MVT_v2f64_r(RetVT, Op0);
3085 default: return Register();
3086 }
3087}
3088
3089// FastEmit functions for ISD::FP_EXTEND.
3090
3091Register fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Register Op0) {
3092 if ((Subtarget->hasFPARMv8())) {
3093 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSHr, RC: &AArch64::FPR32RegClass, Op0);
3094 }
3095 return Register();
3096}
3097
3098Register fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Register Op0) {
3099 if ((Subtarget->hasFPARMv8())) {
3100 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDHr, RC: &AArch64::FPR64RegClass, Op0);
3101 }
3102 return Register();
3103}
3104
3105Register fastEmit_ISD_FP_EXTEND_MVT_f16_r(MVT RetVT, Register Op0) {
3106switch (RetVT.SimpleTy) {
3107 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Op0);
3108 case MVT::f64: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Op0);
3109 default: return Register();
3110}
3111}
3112
3113Register fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
3114 if (RetVT.SimpleTy != MVT::f64)
3115 return Register();
3116 if ((Subtarget->hasFPARMv8())) {
3117 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDSr, RC: &AArch64::FPR64RegClass, Op0);
3118 }
3119 return Register();
3120}
3121
3122Register fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3123 if (RetVT.SimpleTy != MVT::v4f32)
3124 return Register();
3125 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv4i16, RC: &AArch64::FPR128RegClass, Op0);
3126}
3127
3128Register fastEmit_ISD_FP_EXTEND_MVT_v4bf16_r(MVT RetVT, Register Op0) {
3129 if (RetVT.SimpleTy != MVT::v4f32)
3130 return Register();
3131 if ((Subtarget->isNeonAvailable())) {
3132 return fastEmitInst_r(MachineInstOpcode: AArch64::SHLLv4i16, RC: &AArch64::FPR128RegClass, Op0);
3133 }
3134 return Register();
3135}
3136
3137Register fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3138 if (RetVT.SimpleTy != MVT::v2f64)
3139 return Register();
3140 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv2i32, RC: &AArch64::FPR128RegClass, Op0);
3141}
3142
3143Register fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3144 switch (VT.SimpleTy) {
3145 case MVT::f16: return fastEmit_ISD_FP_EXTEND_MVT_f16_r(RetVT, Op0);
3146 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
3147 case MVT::v4f16: return fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
3148 case MVT::v4bf16: return fastEmit_ISD_FP_EXTEND_MVT_v4bf16_r(RetVT, Op0);
3149 case MVT::v2f32: return fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(RetVT, Op0);
3150 default: return Register();
3151 }
3152}
3153
3154// FastEmit functions for ISD::FP_ROUND.
3155
3156Register fastEmit_ISD_FP_ROUND_MVT_f32_MVT_bf16_r(Register Op0) {
3157 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
3158 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVT, RC: &AArch64::FPR16RegClass, Op0);
3159 }
3160 return Register();
3161}
3162
3163Register fastEmit_ISD_FP_ROUND_MVT_f32_MVT_f16_r(Register Op0) {
3164 if ((Subtarget->hasFPARMv8())) {
3165 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHSr, RC: &AArch64::FPR16RegClass, Op0);
3166 }
3167 return Register();
3168}
3169
3170Register fastEmit_ISD_FP_ROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3171switch (RetVT.SimpleTy) {
3172 case MVT::bf16: return fastEmit_ISD_FP_ROUND_MVT_f32_MVT_bf16_r(Op0);
3173 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f32_MVT_f16_r(Op0);
3174 default: return Register();
3175}
3176}
3177
3178Register fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Register Op0) {
3179 if ((Subtarget->hasFPARMv8())) {
3180 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHDr, RC: &AArch64::FPR16RegClass, Op0);
3181 }
3182 return Register();
3183}
3184
3185Register fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Register Op0) {
3186 if ((Subtarget->hasFPARMv8())) {
3187 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSDr, RC: &AArch64::FPR32RegClass, Op0);
3188 }
3189 return Register();
3190}
3191
3192Register fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3193switch (RetVT.SimpleTy) {
3194 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Op0);
3195 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Op0);
3196 default: return Register();
3197}
3198}
3199
3200Register fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Register Op0) {
3201 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
3202}
3203
3204Register fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Register Op0) {
3205 if ((Subtarget->hasBF16()) && (Subtarget->isNeonAvailable())) {
3206 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVTN, RC: &AArch64::FPR64RegClass, Op0);
3207 }
3208 return Register();
3209}
3210
3211Register fastEmit_ISD_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3212switch (RetVT.SimpleTy) {
3213 case MVT::v4f16: return fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Op0);
3214 case MVT::v4bf16: return fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Op0);
3215 default: return Register();
3216}
3217}
3218
3219Register fastEmit_ISD_FP_ROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
3220 if (RetVT.SimpleTy != MVT::v2f32)
3221 return Register();
3222 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
3223}
3224
3225Register fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
3226 switch (VT.SimpleTy) {
3227 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f32_r(RetVT, Op0);
3228 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
3229 case MVT::v4f32: return fastEmit_ISD_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
3230 case MVT::v2f64: return fastEmit_ISD_FP_ROUND_MVT_v2f64_r(RetVT, Op0);
3231 default: return Register();
3232 }
3233}
3234
3235// FastEmit functions for ISD::FP_TO_SINT.
3236
3237Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
3238 if ((Subtarget->hasFPRCVT())) {
3239 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSHr, RC: &AArch64::FPR32RegClass, Op0);
3240 }
3241 if ((Subtarget->hasFullFP16())) {
3242 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
3243 }
3244 return Register();
3245}
3246
3247Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
3248 if ((Subtarget->hasFPRCVT())) {
3249 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDHr, RC: &AArch64::FPR64RegClass, Op0);
3250 }
3251 if ((Subtarget->hasFullFP16())) {
3252 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
3253 }
3254 return Register();
3255}
3256
3257Register fastEmit_ISD_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
3258switch (RetVT.SimpleTy) {
3259 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
3260 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
3261 default: return Register();
3262}
3263}
3264
3265Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
3266 if ((Subtarget->hasFPARMv8())) {
3267 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
3268 }
3269 return Register();
3270}
3271
3272Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
3273 if ((Subtarget->hasFPRCVT())) {
3274 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDSr, RC: &AArch64::FPR64RegClass, Op0);
3275 }
3276 if ((Subtarget->hasFPARMv8())) {
3277 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
3278 }
3279 return Register();
3280}
3281
3282Register fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
3283switch (RetVT.SimpleTy) {
3284 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
3285 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
3286 default: return Register();
3287}
3288}
3289
3290Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
3291 if ((Subtarget->hasFPRCVT())) {
3292 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSDr, RC: &AArch64::FPR32RegClass, Op0);
3293 }
3294 if ((Subtarget->hasFPARMv8())) {
3295 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
3296 }
3297 return Register();
3298}
3299
3300Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
3301 if ((Subtarget->hasFPARMv8())) {
3302 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
3303 }
3304 return Register();
3305}
3306
3307Register fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
3308switch (RetVT.SimpleTy) {
3309 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
3310 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
3311 default: return Register();
3312}
3313}
3314
3315Register fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3316 if (RetVT.SimpleTy != MVT::v4i16)
3317 return Register();
3318 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3319 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
3320 }
3321 return Register();
3322}
3323
3324Register fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3325 if (RetVT.SimpleTy != MVT::v8i16)
3326 return Register();
3327 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3328 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
3329 }
3330 return Register();
3331}
3332
3333Register fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3334 if (RetVT.SimpleTy != MVT::v2i32)
3335 return Register();
3336 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3337 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
3338 }
3339 return Register();
3340}
3341
3342Register fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3343 if (RetVT.SimpleTy != MVT::v4i32)
3344 return Register();
3345 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3346 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
3347 }
3348 return Register();
3349}
3350
3351Register fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3352 if (RetVT.SimpleTy != MVT::v2i64)
3353 return Register();
3354 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3355 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
3356 }
3357 return Register();
3358}
3359
3360Register fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
3361 switch (VT.SimpleTy) {
3362 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
3363 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
3364 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
3365 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
3366 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
3367 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
3368 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
3369 case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0);
3370 default: return Register();
3371 }
3372}
3373
3374// FastEmit functions for ISD::FP_TO_UINT.
3375
3376Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
3377 if ((Subtarget->hasFPRCVT())) {
3378 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSHr, RC: &AArch64::FPR32RegClass, Op0);
3379 }
3380 if ((Subtarget->hasFullFP16())) {
3381 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
3382 }
3383 return Register();
3384}
3385
3386Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
3387 if ((Subtarget->hasFPRCVT())) {
3388 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDHr, RC: &AArch64::FPR64RegClass, Op0);
3389 }
3390 if ((Subtarget->hasFullFP16())) {
3391 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
3392 }
3393 return Register();
3394}
3395
3396Register fastEmit_ISD_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
3397switch (RetVT.SimpleTy) {
3398 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
3399 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
3400 default: return Register();
3401}
3402}
3403
3404Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
3405 if ((Subtarget->hasFPARMv8())) {
3406 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
3407 }
3408 return Register();
3409}
3410
3411Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
3412 if ((Subtarget->hasFPRCVT())) {
3413 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDSr, RC: &AArch64::FPR64RegClass, Op0);
3414 }
3415 if ((Subtarget->hasFPARMv8())) {
3416 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
3417 }
3418 return Register();
3419}
3420
3421Register fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
3422switch (RetVT.SimpleTy) {
3423 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
3424 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
3425 default: return Register();
3426}
3427}
3428
3429Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
3430 if ((Subtarget->hasFPRCVT())) {
3431 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSDr, RC: &AArch64::FPR32RegClass, Op0);
3432 }
3433 if ((Subtarget->hasFPARMv8())) {
3434 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
3435 }
3436 return Register();
3437}
3438
3439Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
3440 if ((Subtarget->hasFPARMv8())) {
3441 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
3442 }
3443 return Register();
3444}
3445
3446Register fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
3447switch (RetVT.SimpleTy) {
3448 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
3449 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
3450 default: return Register();
3451}
3452}
3453
3454Register fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3455 if (RetVT.SimpleTy != MVT::v4i16)
3456 return Register();
3457 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3458 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
3459 }
3460 return Register();
3461}
3462
3463Register fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3464 if (RetVT.SimpleTy != MVT::v8i16)
3465 return Register();
3466 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3467 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
3468 }
3469 return Register();
3470}
3471
3472Register fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3473 if (RetVT.SimpleTy != MVT::v2i32)
3474 return Register();
3475 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3476 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
3477 }
3478 return Register();
3479}
3480
3481Register fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3482 if (RetVT.SimpleTy != MVT::v4i32)
3483 return Register();
3484 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3485 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
3486 }
3487 return Register();
3488}
3489
3490Register fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3491 if (RetVT.SimpleTy != MVT::v2i64)
3492 return Register();
3493 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3494 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
3495 }
3496 return Register();
3497}
3498
3499Register fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
3500 switch (VT.SimpleTy) {
3501 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
3502 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
3503 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
3504 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
3505 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
3506 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
3507 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
3508 case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0);
3509 default: return Register();
3510 }
3511}
3512
3513// FastEmit functions for ISD::FRINT.
3514
3515Register fastEmit_ISD_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
3516 if (RetVT.SimpleTy != MVT::f16)
3517 return Register();
3518 if ((Subtarget->hasFullFP16())) {
3519 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXHr, RC: &AArch64::FPR16RegClass, Op0);
3520 }
3521 return Register();
3522}
3523
3524Register fastEmit_ISD_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
3525 if (RetVT.SimpleTy != MVT::f32)
3526 return Register();
3527 if ((Subtarget->hasFPARMv8())) {
3528 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXSr, RC: &AArch64::FPR32RegClass, Op0);
3529 }
3530 return Register();
3531}
3532
3533Register fastEmit_ISD_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
3534 if (RetVT.SimpleTy != MVT::f64)
3535 return Register();
3536 if ((Subtarget->hasFPARMv8())) {
3537 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXDr, RC: &AArch64::FPR64RegClass, Op0);
3538 }
3539 return Register();
3540}
3541
3542Register fastEmit_ISD_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3543 if (RetVT.SimpleTy != MVT::v4f16)
3544 return Register();
3545 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3546 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f16, RC: &AArch64::FPR64RegClass, Op0);
3547 }
3548 return Register();
3549}
3550
3551Register fastEmit_ISD_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3552 if (RetVT.SimpleTy != MVT::v8f16)
3553 return Register();
3554 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3555 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv8f16, RC: &AArch64::FPR128RegClass, Op0);
3556 }
3557 return Register();
3558}
3559
3560Register fastEmit_ISD_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3561 if (RetVT.SimpleTy != MVT::v2f32)
3562 return Register();
3563 if ((Subtarget->isNeonAvailable())) {
3564 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f32, RC: &AArch64::FPR64RegClass, Op0);
3565 }
3566 return Register();
3567}
3568
3569Register fastEmit_ISD_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3570 if (RetVT.SimpleTy != MVT::v4f32)
3571 return Register();
3572 if ((Subtarget->isNeonAvailable())) {
3573 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f32, RC: &AArch64::FPR128RegClass, Op0);
3574 }
3575 return Register();
3576}
3577
3578Register fastEmit_ISD_FRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3579 if (RetVT.SimpleTy != MVT::v2f64)
3580 return Register();
3581 if ((Subtarget->isNeonAvailable())) {
3582 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f64, RC: &AArch64::FPR128RegClass, Op0);
3583 }
3584 return Register();
3585}
3586
3587Register fastEmit_ISD_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
3588 switch (VT.SimpleTy) {
3589 case MVT::f16: return fastEmit_ISD_FRINT_MVT_f16_r(RetVT, Op0);
3590 case MVT::f32: return fastEmit_ISD_FRINT_MVT_f32_r(RetVT, Op0);
3591 case MVT::f64: return fastEmit_ISD_FRINT_MVT_f64_r(RetVT, Op0);
3592 case MVT::v4f16: return fastEmit_ISD_FRINT_MVT_v4f16_r(RetVT, Op0);
3593 case MVT::v8f16: return fastEmit_ISD_FRINT_MVT_v8f16_r(RetVT, Op0);
3594 case MVT::v2f32: return fastEmit_ISD_FRINT_MVT_v2f32_r(RetVT, Op0);
3595 case MVT::v4f32: return fastEmit_ISD_FRINT_MVT_v4f32_r(RetVT, Op0);
3596 case MVT::v2f64: return fastEmit_ISD_FRINT_MVT_v2f64_r(RetVT, Op0);
3597 default: return Register();
3598 }
3599}
3600
3601// FastEmit functions for ISD::FROUND.
3602
3603Register fastEmit_ISD_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
3604 if (RetVT.SimpleTy != MVT::f16)
3605 return Register();
3606 if ((Subtarget->hasFullFP16())) {
3607 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAHr, RC: &AArch64::FPR16RegClass, Op0);
3608 }
3609 return Register();
3610}
3611
3612Register fastEmit_ISD_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3613 if (RetVT.SimpleTy != MVT::f32)
3614 return Register();
3615 if ((Subtarget->hasFPARMv8())) {
3616 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTASr, RC: &AArch64::FPR32RegClass, Op0);
3617 }
3618 return Register();
3619}
3620
3621Register fastEmit_ISD_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3622 if (RetVT.SimpleTy != MVT::f64)
3623 return Register();
3624 if ((Subtarget->hasFPARMv8())) {
3625 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTADr, RC: &AArch64::FPR64RegClass, Op0);
3626 }
3627 return Register();
3628}
3629
3630Register fastEmit_ISD_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3631 if (RetVT.SimpleTy != MVT::v4f16)
3632 return Register();
3633 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3634 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f16, RC: &AArch64::FPR64RegClass, Op0);
3635 }
3636 return Register();
3637}
3638
3639Register fastEmit_ISD_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
3640 if (RetVT.SimpleTy != MVT::v8f16)
3641 return Register();
3642 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3643 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv8f16, RC: &AArch64::FPR128RegClass, Op0);
3644 }
3645 return Register();
3646}
3647
3648Register fastEmit_ISD_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3649 if (RetVT.SimpleTy != MVT::v2f32)
3650 return Register();
3651 if ((Subtarget->isNeonAvailable())) {
3652 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f32, RC: &AArch64::FPR64RegClass, Op0);
3653 }
3654 return Register();
3655}
3656
3657Register fastEmit_ISD_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3658 if (RetVT.SimpleTy != MVT::v4f32)
3659 return Register();
3660 if ((Subtarget->isNeonAvailable())) {
3661 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f32, RC: &AArch64::FPR128RegClass, Op0);
3662 }
3663 return Register();
3664}
3665
3666Register fastEmit_ISD_FROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
3667 if (RetVT.SimpleTy != MVT::v2f64)
3668 return Register();
3669 if ((Subtarget->isNeonAvailable())) {
3670 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f64, RC: &AArch64::FPR128RegClass, Op0);
3671 }
3672 return Register();
3673}
3674
3675Register fastEmit_ISD_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
3676 switch (VT.SimpleTy) {
3677 case MVT::f16: return fastEmit_ISD_FROUND_MVT_f16_r(RetVT, Op0);
3678 case MVT::f32: return fastEmit_ISD_FROUND_MVT_f32_r(RetVT, Op0);
3679 case MVT::f64: return fastEmit_ISD_FROUND_MVT_f64_r(RetVT, Op0);
3680 case MVT::v4f16: return fastEmit_ISD_FROUND_MVT_v4f16_r(RetVT, Op0);
3681 case MVT::v8f16: return fastEmit_ISD_FROUND_MVT_v8f16_r(RetVT, Op0);
3682 case MVT::v2f32: return fastEmit_ISD_FROUND_MVT_v2f32_r(RetVT, Op0);
3683 case MVT::v4f32: return fastEmit_ISD_FROUND_MVT_v4f32_r(RetVT, Op0);
3684 case MVT::v2f64: return fastEmit_ISD_FROUND_MVT_v2f64_r(RetVT, Op0);
3685 default: return Register();
3686 }
3687}
3688
3689// FastEmit functions for ISD::FROUNDEVEN.
3690
3691Register fastEmit_ISD_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
3692 if (RetVT.SimpleTy != MVT::f16)
3693 return Register();
3694 if ((Subtarget->hasFullFP16())) {
3695 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNHr, RC: &AArch64::FPR16RegClass, Op0);
3696 }
3697 return Register();
3698}
3699
3700Register fastEmit_ISD_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
3701 if (RetVT.SimpleTy != MVT::f32)
3702 return Register();
3703 if ((Subtarget->hasFPARMv8())) {
3704 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNSr, RC: &AArch64::FPR32RegClass, Op0);
3705 }
3706 return Register();
3707}
3708
3709Register fastEmit_ISD_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
3710 if (RetVT.SimpleTy != MVT::f64)
3711 return Register();
3712 if ((Subtarget->hasFPARMv8())) {
3713 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNDr, RC: &AArch64::FPR64RegClass, Op0);
3714 }
3715 return Register();
3716}
3717
3718Register fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
3719 if (RetVT.SimpleTy != MVT::v4f16)
3720 return Register();
3721 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3722 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f16, RC: &AArch64::FPR64RegClass, Op0);
3723 }
3724 return Register();
3725}
3726
3727Register fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
3728 if (RetVT.SimpleTy != MVT::v8f16)
3729 return Register();
3730 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3731 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv8f16, RC: &AArch64::FPR128RegClass, Op0);
3732 }
3733 return Register();
3734}
3735
3736Register fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
3737 if (RetVT.SimpleTy != MVT::v2f32)
3738 return Register();
3739 if ((Subtarget->isNeonAvailable())) {
3740 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f32, RC: &AArch64::FPR64RegClass, Op0);
3741 }
3742 return Register();
3743}
3744
3745Register fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
3746 if (RetVT.SimpleTy != MVT::v4f32)
3747 return Register();
3748 if ((Subtarget->isNeonAvailable())) {
3749 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f32, RC: &AArch64::FPR128RegClass, Op0);
3750 }
3751 return Register();
3752}
3753
3754Register fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, Register Op0) {
3755 if (RetVT.SimpleTy != MVT::v2f64)
3756 return Register();
3757 if ((Subtarget->isNeonAvailable())) {
3758 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f64, RC: &AArch64::FPR128RegClass, Op0);
3759 }
3760 return Register();
3761}
3762
3763Register fastEmit_ISD_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
3764 switch (VT.SimpleTy) {
3765 case MVT::f16: return fastEmit_ISD_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
3766 case MVT::f32: return fastEmit_ISD_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
3767 case MVT::f64: return fastEmit_ISD_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
3768 case MVT::v4f16: return fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
3769 case MVT::v8f16: return fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
3770 case MVT::v2f32: return fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
3771 case MVT::v4f32: return fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
3772 case MVT::v2f64: return fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0);
3773 default: return Register();
3774 }
3775}
3776
3777// FastEmit functions for ISD::FSQRT.
3778
3779Register fastEmit_ISD_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
3780 if (RetVT.SimpleTy != MVT::f16)
3781 return Register();
3782 if ((Subtarget->hasFullFP16())) {
3783 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTHr, RC: &AArch64::FPR16RegClass, Op0);
3784 }
3785 return Register();
3786}
3787
3788Register fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
3789 if (RetVT.SimpleTy != MVT::f32)
3790 return Register();
3791 if ((Subtarget->hasFPARMv8())) {
3792 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTSr, RC: &AArch64::FPR32RegClass, Op0);
3793 }
3794 return Register();
3795}
3796
3797Register fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
3798 if (RetVT.SimpleTy != MVT::f64)
3799 return Register();
3800 if ((Subtarget->hasFPARMv8())) {
3801 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTDr, RC: &AArch64::FPR64RegClass, Op0);
3802 }
3803 return Register();
3804}
3805
3806Register fastEmit_ISD_FSQRT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3807 if (RetVT.SimpleTy != MVT::v4f16)
3808 return Register();
3809 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3810 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f16, RC: &AArch64::FPR64RegClass, Op0);
3811 }
3812 return Register();
3813}
3814
3815Register fastEmit_ISD_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3816 if (RetVT.SimpleTy != MVT::v8f16)
3817 return Register();
3818 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3819 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv8f16, RC: &AArch64::FPR128RegClass, Op0);
3820 }
3821 return Register();
3822}
3823
3824Register fastEmit_ISD_FSQRT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3825 if (RetVT.SimpleTy != MVT::v2f32)
3826 return Register();
3827 if ((Subtarget->isNeonAvailable())) {
3828 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f32, RC: &AArch64::FPR64RegClass, Op0);
3829 }
3830 return Register();
3831}
3832
3833Register fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3834 if (RetVT.SimpleTy != MVT::v4f32)
3835 return Register();
3836 if ((Subtarget->isNeonAvailable())) {
3837 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f32, RC: &AArch64::FPR128RegClass, Op0);
3838 }
3839 return Register();
3840}
3841
3842Register fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3843 if (RetVT.SimpleTy != MVT::v2f64)
3844 return Register();
3845 if ((Subtarget->isNeonAvailable())) {
3846 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f64, RC: &AArch64::FPR128RegClass, Op0);
3847 }
3848 return Register();
3849}
3850
3851Register fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
3852 switch (VT.SimpleTy) {
3853 case MVT::f16: return fastEmit_ISD_FSQRT_MVT_f16_r(RetVT, Op0);
3854 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
3855 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
3856 case MVT::v4f16: return fastEmit_ISD_FSQRT_MVT_v4f16_r(RetVT, Op0);
3857 case MVT::v8f16: return fastEmit_ISD_FSQRT_MVT_v8f16_r(RetVT, Op0);
3858 case MVT::v2f32: return fastEmit_ISD_FSQRT_MVT_v2f32_r(RetVT, Op0);
3859 case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0);
3860 case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0);
3861 default: return Register();
3862 }
3863}
3864
3865// FastEmit functions for ISD::FTRUNC.
3866
3867Register fastEmit_ISD_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
3868 if (RetVT.SimpleTy != MVT::f16)
3869 return Register();
3870 if ((Subtarget->hasFullFP16())) {
3871 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZHr, RC: &AArch64::FPR16RegClass, Op0);
3872 }
3873 return Register();
3874}
3875
3876Register fastEmit_ISD_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
3877 if (RetVT.SimpleTy != MVT::f32)
3878 return Register();
3879 if ((Subtarget->hasFPARMv8())) {
3880 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZSr, RC: &AArch64::FPR32RegClass, Op0);
3881 }
3882 return Register();
3883}
3884
3885Register fastEmit_ISD_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
3886 if (RetVT.SimpleTy != MVT::f64)
3887 return Register();
3888 if ((Subtarget->hasFPARMv8())) {
3889 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZDr, RC: &AArch64::FPR64RegClass, Op0);
3890 }
3891 return Register();
3892}
3893
3894Register fastEmit_ISD_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
3895 if (RetVT.SimpleTy != MVT::v4f16)
3896 return Register();
3897 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3898 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f16, RC: &AArch64::FPR64RegClass, Op0);
3899 }
3900 return Register();
3901}
3902
3903Register fastEmit_ISD_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
3904 if (RetVT.SimpleTy != MVT::v8f16)
3905 return Register();
3906 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3907 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv8f16, RC: &AArch64::FPR128RegClass, Op0);
3908 }
3909 return Register();
3910}
3911
3912Register fastEmit_ISD_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
3913 if (RetVT.SimpleTy != MVT::v2f32)
3914 return Register();
3915 if ((Subtarget->isNeonAvailable())) {
3916 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f32, RC: &AArch64::FPR64RegClass, Op0);
3917 }
3918 return Register();
3919}
3920
3921Register fastEmit_ISD_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
3922 if (RetVT.SimpleTy != MVT::v4f32)
3923 return Register();
3924 if ((Subtarget->isNeonAvailable())) {
3925 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f32, RC: &AArch64::FPR128RegClass, Op0);
3926 }
3927 return Register();
3928}
3929
3930Register fastEmit_ISD_FTRUNC_MVT_v2f64_r(MVT RetVT, Register Op0) {
3931 if (RetVT.SimpleTy != MVT::v2f64)
3932 return Register();
3933 if ((Subtarget->isNeonAvailable())) {
3934 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f64, RC: &AArch64::FPR128RegClass, Op0);
3935 }
3936 return Register();
3937}
3938
3939Register fastEmit_ISD_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
3940 switch (VT.SimpleTy) {
3941 case MVT::f16: return fastEmit_ISD_FTRUNC_MVT_f16_r(RetVT, Op0);
3942 case MVT::f32: return fastEmit_ISD_FTRUNC_MVT_f32_r(RetVT, Op0);
3943 case MVT::f64: return fastEmit_ISD_FTRUNC_MVT_f64_r(RetVT, Op0);
3944 case MVT::v4f16: return fastEmit_ISD_FTRUNC_MVT_v4f16_r(RetVT, Op0);
3945 case MVT::v8f16: return fastEmit_ISD_FTRUNC_MVT_v8f16_r(RetVT, Op0);
3946 case MVT::v2f32: return fastEmit_ISD_FTRUNC_MVT_v2f32_r(RetVT, Op0);
3947 case MVT::v4f32: return fastEmit_ISD_FTRUNC_MVT_v4f32_r(RetVT, Op0);
3948 case MVT::v2f64: return fastEmit_ISD_FTRUNC_MVT_v2f64_r(RetVT, Op0);
3949 default: return Register();
3950 }
3951}
3952
3953// FastEmit functions for ISD::LLROUND.
3954
3955Register fastEmit_ISD_LLROUND_MVT_f16_r(MVT RetVT, Register Op0) {
3956 if (RetVT.SimpleTy != MVT::i64)
3957 return Register();
3958 if ((Subtarget->hasFPRCVT())) {
3959 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDHr, RC: &AArch64::FPR64RegClass, Op0);
3960 }
3961 if ((Subtarget->hasFullFP16())) {
3962 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
3963 }
3964 return Register();
3965}
3966
3967Register fastEmit_ISD_LLROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3968 if (RetVT.SimpleTy != MVT::i64)
3969 return Register();
3970 if ((Subtarget->hasFPRCVT())) {
3971 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDSr, RC: &AArch64::FPR64RegClass, Op0);
3972 }
3973 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
3974}
3975
3976Register fastEmit_ISD_LLROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3977 if (RetVT.SimpleTy != MVT::i64)
3978 return Register();
3979 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
3980}
3981
3982Register fastEmit_ISD_LLROUND_r(MVT VT, MVT RetVT, Register Op0) {
3983 switch (VT.SimpleTy) {
3984 case MVT::f16: return fastEmit_ISD_LLROUND_MVT_f16_r(RetVT, Op0);
3985 case MVT::f32: return fastEmit_ISD_LLROUND_MVT_f32_r(RetVT, Op0);
3986 case MVT::f64: return fastEmit_ISD_LLROUND_MVT_f64_r(RetVT, Op0);
3987 default: return Register();
3988 }
3989}
3990
3991// FastEmit functions for ISD::LROUND.
3992
3993Register fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(Register Op0) {
3994 if ((Subtarget->hasFPRCVT())) {
3995 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASSHr, RC: &AArch64::FPR32RegClass, Op0);
3996 }
3997 if ((Subtarget->hasFullFP16())) {
3998 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWHr, RC: &AArch64::GPR32RegClass, Op0);
3999 }
4000 return Register();
4001}
4002
4003Register fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(Register Op0) {
4004 if ((Subtarget->hasFPRCVT())) {
4005 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDHr, RC: &AArch64::FPR64RegClass, Op0);
4006 }
4007 if ((Subtarget->hasFullFP16())) {
4008 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
4009 }
4010 return Register();
4011}
4012
4013Register fastEmit_ISD_LROUND_MVT_f16_r(MVT RetVT, Register Op0) {
4014switch (RetVT.SimpleTy) {
4015 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(Op0);
4016 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(Op0);
4017 default: return Register();
4018}
4019}
4020
4021Register fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(Register Op0) {
4022 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWSr, RC: &AArch64::GPR32RegClass, Op0);
4023}
4024
4025Register fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(Register Op0) {
4026 if ((Subtarget->hasFPRCVT())) {
4027 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDSr, RC: &AArch64::FPR64RegClass, Op0);
4028 }
4029 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
4030}
4031
4032Register fastEmit_ISD_LROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4033switch (RetVT.SimpleTy) {
4034 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(Op0);
4035 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(Op0);
4036 default: return Register();
4037}
4038}
4039
4040Register fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(Register Op0) {
4041 if ((Subtarget->hasFPRCVT())) {
4042 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASSDr, RC: &AArch64::FPR32RegClass, Op0);
4043 }
4044 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWDr, RC: &AArch64::GPR32RegClass, Op0);
4045}
4046
4047Register fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(Register Op0) {
4048 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
4049}
4050
4051Register fastEmit_ISD_LROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4052switch (RetVT.SimpleTy) {
4053 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(Op0);
4054 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(Op0);
4055 default: return Register();
4056}
4057}
4058
4059Register fastEmit_ISD_LROUND_r(MVT VT, MVT RetVT, Register Op0) {
4060 switch (VT.SimpleTy) {
4061 case MVT::f16: return fastEmit_ISD_LROUND_MVT_f16_r(RetVT, Op0);
4062 case MVT::f32: return fastEmit_ISD_LROUND_MVT_f32_r(RetVT, Op0);
4063 case MVT::f64: return fastEmit_ISD_LROUND_MVT_f64_r(RetVT, Op0);
4064 default: return Register();
4065 }
4066}
4067
4068// FastEmit functions for ISD::SINT_TO_FP.
4069
4070Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
4071 if ((Subtarget->hasFPRCVT())) {
4072 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
4073 }
4074 if ((Subtarget->hasFullFP16())) {
4075 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
4076 }
4077 return Register();
4078}
4079
4080Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
4081 if ((Subtarget->hasFPARMv8())) {
4082 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
4083 }
4084 return Register();
4085}
4086
4087Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
4088 if ((Subtarget->hasFPRCVT())) {
4089 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
4090 }
4091 if ((Subtarget->hasFPARMv8())) {
4092 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
4093 }
4094 return Register();
4095}
4096
4097Register fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
4098switch (RetVT.SimpleTy) {
4099 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
4100 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
4101 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
4102 default: return Register();
4103}
4104}
4105
4106Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
4107 if ((Subtarget->hasFPRCVT())) {
4108 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
4109 }
4110 if ((Subtarget->hasFullFP16())) {
4111 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
4112 }
4113 return Register();
4114}
4115
4116Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
4117 if ((Subtarget->hasFPRCVT())) {
4118 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
4119 }
4120 if ((Subtarget->hasFPARMv8())) {
4121 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
4122 }
4123 return Register();
4124}
4125
4126Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
4127 if ((Subtarget->hasFPARMv8())) {
4128 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
4129 }
4130 return Register();
4131}
4132
4133Register fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
4134switch (RetVT.SimpleTy) {
4135 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
4136 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
4137 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
4138 default: return Register();
4139}
4140}
4141
4142Register fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
4143 if (RetVT.SimpleTy != MVT::v4f16)
4144 return Register();
4145 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4146 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
4147 }
4148 return Register();
4149}
4150
4151Register fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
4152 if (RetVT.SimpleTy != MVT::v8f16)
4153 return Register();
4154 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4155 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
4156 }
4157 return Register();
4158}
4159
4160Register fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
4161 if (RetVT.SimpleTy != MVT::v2f32)
4162 return Register();
4163 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4164 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
4165 }
4166 return Register();
4167}
4168
4169Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
4170 if (RetVT.SimpleTy != MVT::v4f32)
4171 return Register();
4172 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4173 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
4174 }
4175 return Register();
4176}
4177
4178Register fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
4179 if (RetVT.SimpleTy != MVT::v2f64)
4180 return Register();
4181 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4182 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
4183 }
4184 return Register();
4185}
4186
4187Register fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
4188 switch (VT.SimpleTy) {
4189 case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
4190 case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
4191 case MVT::v4i16: return fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
4192 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
4193 case MVT::v2i32: return fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
4194 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
4195 case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
4196 default: return Register();
4197 }
4198}
4199
4200// FastEmit functions for ISD::SPLAT_VECTOR.
4201
4202Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(Register Op0) {
4203 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4204 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_B, RC: &AArch64::ZPRRegClass, Op0);
4205 }
4206 return Register();
4207}
4208
4209Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(Register Op0) {
4210 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4211 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_H, RC: &AArch64::ZPRRegClass, Op0);
4212 }
4213 return Register();
4214}
4215
4216Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(Register Op0) {
4217 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4218 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_S, RC: &AArch64::ZPRRegClass, Op0);
4219 }
4220 return Register();
4221}
4222
4223Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(MVT RetVT, Register Op0) {
4224switch (RetVT.SimpleTy) {
4225 case MVT::nxv16i8: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(Op0);
4226 case MVT::nxv8i16: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(Op0);
4227 case MVT::nxv4i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(Op0);
4228 default: return Register();
4229}
4230}
4231
4232Register fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(MVT RetVT, Register Op0) {
4233 if (RetVT.SimpleTy != MVT::nxv2i64)
4234 return Register();
4235 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4236 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_D, RC: &AArch64::ZPRRegClass, Op0);
4237 }
4238 return Register();
4239}
4240
4241Register fastEmit_ISD_SPLAT_VECTOR_r(MVT VT, MVT RetVT, Register Op0) {
4242 switch (VT.SimpleTy) {
4243 case MVT::i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(RetVT, Op0);
4244 case MVT::i64: return fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(RetVT, Op0);
4245 default: return Register();
4246 }
4247}
4248
4249// FastEmit functions for ISD::STRICT_FCEIL.
4250
4251Register fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
4252 if (RetVT.SimpleTy != MVT::f16)
4253 return Register();
4254 if ((Subtarget->hasFullFP16())) {
4255 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPHr, RC: &AArch64::FPR16RegClass, Op0);
4256 }
4257 return Register();
4258}
4259
4260Register fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
4261 if (RetVT.SimpleTy != MVT::f32)
4262 return Register();
4263 if ((Subtarget->hasFPARMv8())) {
4264 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPSr, RC: &AArch64::FPR32RegClass, Op0);
4265 }
4266 return Register();
4267}
4268
4269Register fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
4270 if (RetVT.SimpleTy != MVT::f64)
4271 return Register();
4272 if ((Subtarget->hasFPARMv8())) {
4273 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPDr, RC: &AArch64::FPR64RegClass, Op0);
4274 }
4275 return Register();
4276}
4277
4278Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
4279 if (RetVT.SimpleTy != MVT::v4f16)
4280 return Register();
4281 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4282 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f16, RC: &AArch64::FPR64RegClass, Op0);
4283 }
4284 return Register();
4285}
4286
4287Register fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
4288 if (RetVT.SimpleTy != MVT::v8f16)
4289 return Register();
4290 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4291 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv8f16, RC: &AArch64::FPR128RegClass, Op0);
4292 }
4293 return Register();
4294}
4295
4296Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
4297 if (RetVT.SimpleTy != MVT::v2f32)
4298 return Register();
4299 if ((Subtarget->isNeonAvailable())) {
4300 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f32, RC: &AArch64::FPR64RegClass, Op0);
4301 }
4302 return Register();
4303}
4304
4305Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
4306 if (RetVT.SimpleTy != MVT::v4f32)
4307 return Register();
4308 if ((Subtarget->isNeonAvailable())) {
4309 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f32, RC: &AArch64::FPR128RegClass, Op0);
4310 }
4311 return Register();
4312}
4313
4314Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(MVT RetVT, Register Op0) {
4315 if (RetVT.SimpleTy != MVT::v2f64)
4316 return Register();
4317 if ((Subtarget->isNeonAvailable())) {
4318 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f64, RC: &AArch64::FPR128RegClass, Op0);
4319 }
4320 return Register();
4321}
4322
4323Register fastEmit_ISD_STRICT_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
4324 switch (VT.SimpleTy) {
4325 case MVT::f16: return fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(RetVT, Op0);
4326 case MVT::f32: return fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(RetVT, Op0);
4327 case MVT::f64: return fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(RetVT, Op0);
4328 case MVT::v4f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(RetVT, Op0);
4329 case MVT::v8f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(RetVT, Op0);
4330 case MVT::v2f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(RetVT, Op0);
4331 case MVT::v4f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(RetVT, Op0);
4332 case MVT::v2f64: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(RetVT, Op0);
4333 default: return Register();
4334 }
4335}
4336
4337// FastEmit functions for ISD::STRICT_FFLOOR.
4338
4339Register fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
4340 if (RetVT.SimpleTy != MVT::f16)
4341 return Register();
4342 if ((Subtarget->hasFullFP16())) {
4343 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMHr, RC: &AArch64::FPR16RegClass, Op0);
4344 }
4345 return Register();
4346}
4347
4348Register fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
4349 if (RetVT.SimpleTy != MVT::f32)
4350 return Register();
4351 if ((Subtarget->hasFPARMv8())) {
4352 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMSr, RC: &AArch64::FPR32RegClass, Op0);
4353 }
4354 return Register();
4355}
4356
4357Register fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
4358 if (RetVT.SimpleTy != MVT::f64)
4359 return Register();
4360 if ((Subtarget->hasFPARMv8())) {
4361 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMDr, RC: &AArch64::FPR64RegClass, Op0);
4362 }
4363 return Register();
4364}
4365
4366Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
4367 if (RetVT.SimpleTy != MVT::v4f16)
4368 return Register();
4369 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4370 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f16, RC: &AArch64::FPR64RegClass, Op0);
4371 }
4372 return Register();
4373}
4374
4375Register fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
4376 if (RetVT.SimpleTy != MVT::v8f16)
4377 return Register();
4378 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4379 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv8f16, RC: &AArch64::FPR128RegClass, Op0);
4380 }
4381 return Register();
4382}
4383
4384Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
4385 if (RetVT.SimpleTy != MVT::v2f32)
4386 return Register();
4387 if ((Subtarget->isNeonAvailable())) {
4388 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f32, RC: &AArch64::FPR64RegClass, Op0);
4389 }
4390 return Register();
4391}
4392
4393Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
4394 if (RetVT.SimpleTy != MVT::v4f32)
4395 return Register();
4396 if ((Subtarget->isNeonAvailable())) {
4397 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f32, RC: &AArch64::FPR128RegClass, Op0);
4398 }
4399 return Register();
4400}
4401
4402Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(MVT RetVT, Register Op0) {
4403 if (RetVT.SimpleTy != MVT::v2f64)
4404 return Register();
4405 if ((Subtarget->isNeonAvailable())) {
4406 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f64, RC: &AArch64::FPR128RegClass, Op0);
4407 }
4408 return Register();
4409}
4410
4411Register fastEmit_ISD_STRICT_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
4412 switch (VT.SimpleTy) {
4413 case MVT::f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(RetVT, Op0);
4414 case MVT::f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(RetVT, Op0);
4415 case MVT::f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(RetVT, Op0);
4416 case MVT::v4f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(RetVT, Op0);
4417 case MVT::v8f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(RetVT, Op0);
4418 case MVT::v2f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(RetVT, Op0);
4419 case MVT::v4f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(RetVT, Op0);
4420 case MVT::v2f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(RetVT, Op0);
4421 default: return Register();
4422 }
4423}
4424
4425// FastEmit functions for ISD::STRICT_FNEARBYINT.
4426
4427Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
4428 if (RetVT.SimpleTy != MVT::f16)
4429 return Register();
4430 if ((Subtarget->hasFullFP16())) {
4431 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIHr, RC: &AArch64::FPR16RegClass, Op0);
4432 }
4433 return Register();
4434}
4435
4436Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
4437 if (RetVT.SimpleTy != MVT::f32)
4438 return Register();
4439 if ((Subtarget->hasFPARMv8())) {
4440 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTISr, RC: &AArch64::FPR32RegClass, Op0);
4441 }
4442 return Register();
4443}
4444
4445Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
4446 if (RetVT.SimpleTy != MVT::f64)
4447 return Register();
4448 if ((Subtarget->hasFPARMv8())) {
4449 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIDr, RC: &AArch64::FPR64RegClass, Op0);
4450 }
4451 return Register();
4452}
4453
4454Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4455 if (RetVT.SimpleTy != MVT::v4f16)
4456 return Register();
4457 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4458 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f16, RC: &AArch64::FPR64RegClass, Op0);
4459 }
4460 return Register();
4461}
4462
4463Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4464 if (RetVT.SimpleTy != MVT::v8f16)
4465 return Register();
4466 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4467 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv8f16, RC: &AArch64::FPR128RegClass, Op0);
4468 }
4469 return Register();
4470}
4471
4472Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4473 if (RetVT.SimpleTy != MVT::v2f32)
4474 return Register();
4475 if ((Subtarget->isNeonAvailable())) {
4476 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f32, RC: &AArch64::FPR64RegClass, Op0);
4477 }
4478 return Register();
4479}
4480
4481Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4482 if (RetVT.SimpleTy != MVT::v4f32)
4483 return Register();
4484 if ((Subtarget->isNeonAvailable())) {
4485 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f32, RC: &AArch64::FPR128RegClass, Op0);
4486 }
4487 return Register();
4488}
4489
4490Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4491 if (RetVT.SimpleTy != MVT::v2f64)
4492 return Register();
4493 if ((Subtarget->isNeonAvailable())) {
4494 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f64, RC: &AArch64::FPR128RegClass, Op0);
4495 }
4496 return Register();
4497}
4498
4499Register fastEmit_ISD_STRICT_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
4500 switch (VT.SimpleTy) {
4501 case MVT::f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(RetVT, Op0);
4502 case MVT::f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(RetVT, Op0);
4503 case MVT::f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(RetVT, Op0);
4504 case MVT::v4f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(RetVT, Op0);
4505 case MVT::v8f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(RetVT, Op0);
4506 case MVT::v2f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(RetVT, Op0);
4507 case MVT::v4f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(RetVT, Op0);
4508 case MVT::v2f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(RetVT, Op0);
4509 default: return Register();
4510 }
4511}
4512
4513// FastEmit functions for ISD::STRICT_FP_EXTEND.
4514
4515Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(Register Op0) {
4516 if ((Subtarget->hasFPARMv8())) {
4517 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSHr, RC: &AArch64::FPR32RegClass, Op0);
4518 }
4519 return Register();
4520}
4521
4522Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(Register Op0) {
4523 if ((Subtarget->hasFPARMv8())) {
4524 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDHr, RC: &AArch64::FPR64RegClass, Op0);
4525 }
4526 return Register();
4527}
4528
4529Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(MVT RetVT, Register Op0) {
4530switch (RetVT.SimpleTy) {
4531 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(Op0);
4532 case MVT::f64: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(Op0);
4533 default: return Register();
4534}
4535}
4536
4537Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
4538 if (RetVT.SimpleTy != MVT::f64)
4539 return Register();
4540 if ((Subtarget->hasFPARMv8())) {
4541 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDSr, RC: &AArch64::FPR64RegClass, Op0);
4542 }
4543 return Register();
4544}
4545
4546Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
4547 if (RetVT.SimpleTy != MVT::v4f32)
4548 return Register();
4549 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv4i16, RC: &AArch64::FPR128RegClass, Op0);
4550}
4551
4552Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4bf16_r(MVT RetVT, Register Op0) {
4553 if (RetVT.SimpleTy != MVT::v4f32)
4554 return Register();
4555 if ((Subtarget->isNeonAvailable())) {
4556 return fastEmitInst_r(MachineInstOpcode: AArch64::SHLLv4i16, RC: &AArch64::FPR128RegClass, Op0);
4557 }
4558 return Register();
4559}
4560
4561Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(MVT RetVT, Register Op0) {
4562 if (RetVT.SimpleTy != MVT::v2f64)
4563 return Register();
4564 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv2i32, RC: &AArch64::FPR128RegClass, Op0);
4565}
4566
4567Register fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
4568 switch (VT.SimpleTy) {
4569 case MVT::f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(RetVT, Op0);
4570 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0);
4571 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
4572 case MVT::v4bf16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4bf16_r(RetVT, Op0);
4573 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(RetVT, Op0);
4574 default: return Register();
4575 }
4576}
4577
4578// FastEmit functions for ISD::STRICT_FP_ROUND.
4579
4580Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_bf16_r(Register Op0) {
4581 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
4582 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVT, RC: &AArch64::FPR16RegClass, Op0);
4583 }
4584 return Register();
4585}
4586
4587Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_f16_r(Register Op0) {
4588 if ((Subtarget->hasFPARMv8())) {
4589 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHSr, RC: &AArch64::FPR16RegClass, Op0);
4590 }
4591 return Register();
4592}
4593
4594Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4595switch (RetVT.SimpleTy) {
4596 case MVT::bf16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_bf16_r(Op0);
4597 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_f16_r(Op0);
4598 default: return Register();
4599}
4600}
4601
4602Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(Register Op0) {
4603 if ((Subtarget->hasFPARMv8())) {
4604 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHDr, RC: &AArch64::FPR16RegClass, Op0);
4605 }
4606 return Register();
4607}
4608
4609Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(Register Op0) {
4610 if ((Subtarget->hasFPARMv8())) {
4611 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSDr, RC: &AArch64::FPR32RegClass, Op0);
4612 }
4613 return Register();
4614}
4615
4616Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4617switch (RetVT.SimpleTy) {
4618 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(Op0);
4619 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(Op0);
4620 default: return Register();
4621}
4622}
4623
4624Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Register Op0) {
4625 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
4626}
4627
4628Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Register Op0) {
4629 if ((Subtarget->hasBF16()) && (Subtarget->isNeonAvailable())) {
4630 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVTN, RC: &AArch64::FPR64RegClass, Op0);
4631 }
4632 return Register();
4633}
4634
4635Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
4636switch (RetVT.SimpleTy) {
4637 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Op0);
4638 case MVT::v4bf16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Op0);
4639 default: return Register();
4640}
4641}
4642
4643Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
4644 if (RetVT.SimpleTy != MVT::v2f32)
4645 return Register();
4646 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
4647}
4648
4649Register fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
4650 switch (VT.SimpleTy) {
4651 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(RetVT, Op0);
4652 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0);
4653 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
4654 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(RetVT, Op0);
4655 default: return Register();
4656 }
4657}
4658
4659// FastEmit functions for ISD::STRICT_FP_TO_SINT.
4660
4661Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
4662 if ((Subtarget->hasFPRCVT())) {
4663 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSHr, RC: &AArch64::FPR32RegClass, Op0);
4664 }
4665 if ((Subtarget->hasFullFP16())) {
4666 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
4667 }
4668 return Register();
4669}
4670
4671Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
4672 if ((Subtarget->hasFPRCVT())) {
4673 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDHr, RC: &AArch64::FPR64RegClass, Op0);
4674 }
4675 if ((Subtarget->hasFullFP16())) {
4676 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
4677 }
4678 return Register();
4679}
4680
4681Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
4682switch (RetVT.SimpleTy) {
4683 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
4684 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
4685 default: return Register();
4686}
4687}
4688
4689Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
4690 if ((Subtarget->hasFPARMv8())) {
4691 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
4692 }
4693 return Register();
4694}
4695
4696Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
4697 if ((Subtarget->hasFPRCVT())) {
4698 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDSr, RC: &AArch64::FPR64RegClass, Op0);
4699 }
4700 if ((Subtarget->hasFPARMv8())) {
4701 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
4702 }
4703 return Register();
4704}
4705
4706Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
4707switch (RetVT.SimpleTy) {
4708 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
4709 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
4710 default: return Register();
4711}
4712}
4713
4714Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
4715 if ((Subtarget->hasFPRCVT())) {
4716 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSDr, RC: &AArch64::FPR32RegClass, Op0);
4717 }
4718 if ((Subtarget->hasFPARMv8())) {
4719 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
4720 }
4721 return Register();
4722}
4723
4724Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
4725 if ((Subtarget->hasFPARMv8())) {
4726 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
4727 }
4728 return Register();
4729}
4730
4731Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
4732switch (RetVT.SimpleTy) {
4733 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
4734 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
4735 default: return Register();
4736}
4737}
4738
4739Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4740 if (RetVT.SimpleTy != MVT::v4i16)
4741 return Register();
4742 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4743 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
4744 }
4745 return Register();
4746}
4747
4748Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4749 if (RetVT.SimpleTy != MVT::v8i16)
4750 return Register();
4751 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4752 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
4753 }
4754 return Register();
4755}
4756
4757Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4758 if (RetVT.SimpleTy != MVT::v2i32)
4759 return Register();
4760 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4761 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
4762 }
4763 return Register();
4764}
4765
4766Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4767 if (RetVT.SimpleTy != MVT::v4i32)
4768 return Register();
4769 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4770 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
4771 }
4772 return Register();
4773}
4774
4775Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4776 if (RetVT.SimpleTy != MVT::v2i64)
4777 return Register();
4778 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4779 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
4780 }
4781 return Register();
4782}
4783
4784Register fastEmit_ISD_STRICT_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
4785 switch (VT.SimpleTy) {
4786 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
4787 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
4788 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
4789 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
4790 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
4791 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
4792 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
4793 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0);
4794 default: return Register();
4795 }
4796}
4797
4798// FastEmit functions for ISD::STRICT_FP_TO_UINT.
4799
4800Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
4801 if ((Subtarget->hasFPRCVT())) {
4802 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSHr, RC: &AArch64::FPR32RegClass, Op0);
4803 }
4804 if ((Subtarget->hasFullFP16())) {
4805 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
4806 }
4807 return Register();
4808}
4809
4810Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
4811 if ((Subtarget->hasFPRCVT())) {
4812 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDHr, RC: &AArch64::FPR64RegClass, Op0);
4813 }
4814 if ((Subtarget->hasFullFP16())) {
4815 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
4816 }
4817 return Register();
4818}
4819
4820Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
4821switch (RetVT.SimpleTy) {
4822 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
4823 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
4824 default: return Register();
4825}
4826}
4827
4828Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
4829 if ((Subtarget->hasFPARMv8())) {
4830 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
4831 }
4832 return Register();
4833}
4834
4835Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
4836 if ((Subtarget->hasFPRCVT())) {
4837 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDSr, RC: &AArch64::FPR64RegClass, Op0);
4838 }
4839 if ((Subtarget->hasFPARMv8())) {
4840 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
4841 }
4842 return Register();
4843}
4844
4845Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
4846switch (RetVT.SimpleTy) {
4847 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
4848 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
4849 default: return Register();
4850}
4851}
4852
4853Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
4854 if ((Subtarget->hasFPRCVT())) {
4855 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSDr, RC: &AArch64::FPR32RegClass, Op0);
4856 }
4857 if ((Subtarget->hasFPARMv8())) {
4858 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
4859 }
4860 return Register();
4861}
4862
4863Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
4864 if ((Subtarget->hasFPARMv8())) {
4865 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
4866 }
4867 return Register();
4868}
4869
4870Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
4871switch (RetVT.SimpleTy) {
4872 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
4873 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
4874 default: return Register();
4875}
4876}
4877
4878Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4879 if (RetVT.SimpleTy != MVT::v4i16)
4880 return Register();
4881 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4882 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
4883 }
4884 return Register();
4885}
4886
4887Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4888 if (RetVT.SimpleTy != MVT::v8i16)
4889 return Register();
4890 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4891 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
4892 }
4893 return Register();
4894}
4895
4896Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4897 if (RetVT.SimpleTy != MVT::v2i32)
4898 return Register();
4899 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4900 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
4901 }
4902 return Register();
4903}
4904
4905Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4906 if (RetVT.SimpleTy != MVT::v4i32)
4907 return Register();
4908 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4909 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
4910 }
4911 return Register();
4912}
4913
4914Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4915 if (RetVT.SimpleTy != MVT::v2i64)
4916 return Register();
4917 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4918 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
4919 }
4920 return Register();
4921}
4922
4923Register fastEmit_ISD_STRICT_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
4924 switch (VT.SimpleTy) {
4925 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
4926 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
4927 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
4928 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
4929 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
4930 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
4931 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
4932 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0);
4933 default: return Register();
4934 }
4935}
4936
4937// FastEmit functions for ISD::STRICT_FRINT.
4938
4939Register fastEmit_ISD_STRICT_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
4940 if (RetVT.SimpleTy != MVT::f16)
4941 return Register();
4942 if ((Subtarget->hasFullFP16())) {
4943 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXHr, RC: &AArch64::FPR16RegClass, Op0);
4944 }
4945 return Register();
4946}
4947
4948Register fastEmit_ISD_STRICT_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
4949 if (RetVT.SimpleTy != MVT::f32)
4950 return Register();
4951 if ((Subtarget->hasFPARMv8())) {
4952 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXSr, RC: &AArch64::FPR32RegClass, Op0);
4953 }
4954 return Register();
4955}
4956
4957Register fastEmit_ISD_STRICT_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
4958 if (RetVT.SimpleTy != MVT::f64)
4959 return Register();
4960 if ((Subtarget->hasFPARMv8())) {
4961 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXDr, RC: &AArch64::FPR64RegClass, Op0);
4962 }
4963 return Register();
4964}
4965
4966Register fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4967 if (RetVT.SimpleTy != MVT::v4f16)
4968 return Register();
4969 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4970 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f16, RC: &AArch64::FPR64RegClass, Op0);
4971 }
4972 return Register();
4973}
4974
4975Register fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4976 if (RetVT.SimpleTy != MVT::v8f16)
4977 return Register();
4978 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4979 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv8f16, RC: &AArch64::FPR128RegClass, Op0);
4980 }
4981 return Register();
4982}
4983
4984Register fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4985 if (RetVT.SimpleTy != MVT::v2f32)
4986 return Register();
4987 if ((Subtarget->isNeonAvailable())) {
4988 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f32, RC: &AArch64::FPR64RegClass, Op0);
4989 }
4990 return Register();
4991}
4992
4993Register fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4994 if (RetVT.SimpleTy != MVT::v4f32)
4995 return Register();
4996 if ((Subtarget->isNeonAvailable())) {
4997 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f32, RC: &AArch64::FPR128RegClass, Op0);
4998 }
4999 return Register();
5000}
5001
5002Register fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
5003 if (RetVT.SimpleTy != MVT::v2f64)
5004 return Register();
5005 if ((Subtarget->isNeonAvailable())) {
5006 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f64, RC: &AArch64::FPR128RegClass, Op0);
5007 }
5008 return Register();
5009}
5010
5011Register fastEmit_ISD_STRICT_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
5012 switch (VT.SimpleTy) {
5013 case MVT::f16: return fastEmit_ISD_STRICT_FRINT_MVT_f16_r(RetVT, Op0);
5014 case MVT::f32: return fastEmit_ISD_STRICT_FRINT_MVT_f32_r(RetVT, Op0);
5015 case MVT::f64: return fastEmit_ISD_STRICT_FRINT_MVT_f64_r(RetVT, Op0);
5016 case MVT::v4f16: return fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(RetVT, Op0);
5017 case MVT::v8f16: return fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(RetVT, Op0);
5018 case MVT::v2f32: return fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(RetVT, Op0);
5019 case MVT::v4f32: return fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(RetVT, Op0);
5020 case MVT::v2f64: return fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(RetVT, Op0);
5021 default: return Register();
5022 }
5023}
5024
5025// FastEmit functions for ISD::STRICT_FROUND.
5026
5027Register fastEmit_ISD_STRICT_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5028 if (RetVT.SimpleTy != MVT::f16)
5029 return Register();
5030 if ((Subtarget->hasFullFP16())) {
5031 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAHr, RC: &AArch64::FPR16RegClass, Op0);
5032 }
5033 return Register();
5034}
5035
5036Register fastEmit_ISD_STRICT_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5037 if (RetVT.SimpleTy != MVT::f32)
5038 return Register();
5039 if ((Subtarget->hasFPARMv8())) {
5040 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTASr, RC: &AArch64::FPR32RegClass, Op0);
5041 }
5042 return Register();
5043}
5044
5045Register fastEmit_ISD_STRICT_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5046 if (RetVT.SimpleTy != MVT::f64)
5047 return Register();
5048 if ((Subtarget->hasFPARMv8())) {
5049 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTADr, RC: &AArch64::FPR64RegClass, Op0);
5050 }
5051 return Register();
5052}
5053
5054Register fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
5055 if (RetVT.SimpleTy != MVT::v4f16)
5056 return Register();
5057 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5058 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f16, RC: &AArch64::FPR64RegClass, Op0);
5059 }
5060 return Register();
5061}
5062
5063Register fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
5064 if (RetVT.SimpleTy != MVT::v8f16)
5065 return Register();
5066 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5067 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv8f16, RC: &AArch64::FPR128RegClass, Op0);
5068 }
5069 return Register();
5070}
5071
5072Register fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
5073 if (RetVT.SimpleTy != MVT::v2f32)
5074 return Register();
5075 if ((Subtarget->isNeonAvailable())) {
5076 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f32, RC: &AArch64::FPR64RegClass, Op0);
5077 }
5078 return Register();
5079}
5080
5081Register fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
5082 if (RetVT.SimpleTy != MVT::v4f32)
5083 return Register();
5084 if ((Subtarget->isNeonAvailable())) {
5085 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f32, RC: &AArch64::FPR128RegClass, Op0);
5086 }
5087 return Register();
5088}
5089
5090Register fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
5091 if (RetVT.SimpleTy != MVT::v2f64)
5092 return Register();
5093 if ((Subtarget->isNeonAvailable())) {
5094 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f64, RC: &AArch64::FPR128RegClass, Op0);
5095 }
5096 return Register();
5097}
5098
5099Register fastEmit_ISD_STRICT_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
5100 switch (VT.SimpleTy) {
5101 case MVT::f16: return fastEmit_ISD_STRICT_FROUND_MVT_f16_r(RetVT, Op0);
5102 case MVT::f32: return fastEmit_ISD_STRICT_FROUND_MVT_f32_r(RetVT, Op0);
5103 case MVT::f64: return fastEmit_ISD_STRICT_FROUND_MVT_f64_r(RetVT, Op0);
5104 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(RetVT, Op0);
5105 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(RetVT, Op0);
5106 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(RetVT, Op0);
5107 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(RetVT, Op0);
5108 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(RetVT, Op0);
5109 default: return Register();
5110 }
5111}
5112
5113// FastEmit functions for ISD::STRICT_FROUNDEVEN.
5114
5115Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
5116 if (RetVT.SimpleTy != MVT::f16)
5117 return Register();
5118 if ((Subtarget->hasFullFP16())) {
5119 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNHr, RC: &AArch64::FPR16RegClass, Op0);
5120 }
5121 return Register();
5122}
5123
5124Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
5125 if (RetVT.SimpleTy != MVT::f32)
5126 return Register();
5127 if ((Subtarget->hasFPARMv8())) {
5128 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNSr, RC: &AArch64::FPR32RegClass, Op0);
5129 }
5130 return Register();
5131}
5132
5133Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
5134 if (RetVT.SimpleTy != MVT::f64)
5135 return Register();
5136 if ((Subtarget->hasFPARMv8())) {
5137 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNDr, RC: &AArch64::FPR64RegClass, Op0);
5138 }
5139 return Register();
5140}
5141
5142Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
5143 if (RetVT.SimpleTy != MVT::v4f16)
5144 return Register();
5145 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5146 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f16, RC: &AArch64::FPR64RegClass, Op0);
5147 }
5148 return Register();
5149}
5150
5151Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
5152 if (RetVT.SimpleTy != MVT::v8f16)
5153 return Register();
5154 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5155 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv8f16, RC: &AArch64::FPR128RegClass, Op0);
5156 }
5157 return Register();
5158}
5159
5160Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
5161 if (RetVT.SimpleTy != MVT::v2f32)
5162 return Register();
5163 if ((Subtarget->isNeonAvailable())) {
5164 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f32, RC: &AArch64::FPR64RegClass, Op0);
5165 }
5166 return Register();
5167}
5168
5169Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
5170 if (RetVT.SimpleTy != MVT::v4f32)
5171 return Register();
5172 if ((Subtarget->isNeonAvailable())) {
5173 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f32, RC: &AArch64::FPR128RegClass, Op0);
5174 }
5175 return Register();
5176}
5177
5178Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, Register Op0) {
5179 if (RetVT.SimpleTy != MVT::v2f64)
5180 return Register();
5181 if ((Subtarget->isNeonAvailable())) {
5182 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f64, RC: &AArch64::FPR128RegClass, Op0);
5183 }
5184 return Register();
5185}
5186
5187Register fastEmit_ISD_STRICT_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
5188 switch (VT.SimpleTy) {
5189 case MVT::f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
5190 case MVT::f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
5191 case MVT::f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
5192 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
5193 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
5194 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
5195 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
5196 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0);
5197 default: return Register();
5198 }
5199}
5200
5201// FastEmit functions for ISD::STRICT_FSQRT.
5202
5203Register fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
5204 if (RetVT.SimpleTy != MVT::f16)
5205 return Register();
5206 if ((Subtarget->hasFullFP16())) {
5207 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTHr, RC: &AArch64::FPR16RegClass, Op0);
5208 }
5209 return Register();
5210}
5211
5212Register fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
5213 if (RetVT.SimpleTy != MVT::f32)
5214 return Register();
5215 if ((Subtarget->hasFPARMv8())) {
5216 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTSr, RC: &AArch64::FPR32RegClass, Op0);
5217 }
5218 return Register();
5219}
5220
5221Register fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
5222 if (RetVT.SimpleTy != MVT::f64)
5223 return Register();
5224 if ((Subtarget->hasFPARMv8())) {
5225 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTDr, RC: &AArch64::FPR64RegClass, Op0);
5226 }
5227 return Register();
5228}
5229
5230Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(MVT RetVT, Register Op0) {
5231 if (RetVT.SimpleTy != MVT::v4f16)
5232 return Register();
5233 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5234 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f16, RC: &AArch64::FPR64RegClass, Op0);
5235 }
5236 return Register();
5237}
5238
5239Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
5240 if (RetVT.SimpleTy != MVT::v8f16)
5241 return Register();
5242 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5243 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv8f16, RC: &AArch64::FPR128RegClass, Op0);
5244 }
5245 return Register();
5246}
5247
5248Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(MVT RetVT, Register Op0) {
5249 if (RetVT.SimpleTy != MVT::v2f32)
5250 return Register();
5251 if ((Subtarget->isNeonAvailable())) {
5252 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f32, RC: &AArch64::FPR64RegClass, Op0);
5253 }
5254 return Register();
5255}
5256
5257Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
5258 if (RetVT.SimpleTy != MVT::v4f32)
5259 return Register();
5260 if ((Subtarget->isNeonAvailable())) {
5261 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f32, RC: &AArch64::FPR128RegClass, Op0);
5262 }
5263 return Register();
5264}
5265
5266Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
5267 if (RetVT.SimpleTy != MVT::v2f64)
5268 return Register();
5269 if ((Subtarget->isNeonAvailable())) {
5270 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f64, RC: &AArch64::FPR128RegClass, Op0);
5271 }
5272 return Register();
5273}
5274
5275Register fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
5276 switch (VT.SimpleTy) {
5277 case MVT::f16: return fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(RetVT, Op0);
5278 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0);
5279 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0);
5280 case MVT::v4f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(RetVT, Op0);
5281 case MVT::v8f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(RetVT, Op0);
5282 case MVT::v2f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(RetVT, Op0);
5283 case MVT::v4f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(RetVT, Op0);
5284 case MVT::v2f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(RetVT, Op0);
5285 default: return Register();
5286 }
5287}
5288
5289// FastEmit functions for ISD::STRICT_FTRUNC.
5290
5291Register fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
5292 if (RetVT.SimpleTy != MVT::f16)
5293 return Register();
5294 if ((Subtarget->hasFullFP16())) {
5295 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZHr, RC: &AArch64::FPR16RegClass, Op0);
5296 }
5297 return Register();
5298}
5299
5300Register fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
5301 if (RetVT.SimpleTy != MVT::f32)
5302 return Register();
5303 if ((Subtarget->hasFPARMv8())) {
5304 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZSr, RC: &AArch64::FPR32RegClass, Op0);
5305 }
5306 return Register();
5307}
5308
5309Register fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
5310 if (RetVT.SimpleTy != MVT::f64)
5311 return Register();
5312 if ((Subtarget->hasFPARMv8())) {
5313 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZDr, RC: &AArch64::FPR64RegClass, Op0);
5314 }
5315 return Register();
5316}
5317
5318Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
5319 if (RetVT.SimpleTy != MVT::v4f16)
5320 return Register();
5321 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5322 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f16, RC: &AArch64::FPR64RegClass, Op0);
5323 }
5324 return Register();
5325}
5326
5327Register fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
5328 if (RetVT.SimpleTy != MVT::v8f16)
5329 return Register();
5330 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5331 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv8f16, RC: &AArch64::FPR128RegClass, Op0);
5332 }
5333 return Register();
5334}
5335
5336Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
5337 if (RetVT.SimpleTy != MVT::v2f32)
5338 return Register();
5339 if ((Subtarget->isNeonAvailable())) {
5340 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f32, RC: &AArch64::FPR64RegClass, Op0);
5341 }
5342 return Register();
5343}
5344
5345Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
5346 if (RetVT.SimpleTy != MVT::v4f32)
5347 return Register();
5348 if ((Subtarget->isNeonAvailable())) {
5349 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f32, RC: &AArch64::FPR128RegClass, Op0);
5350 }
5351 return Register();
5352}
5353
5354Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(MVT RetVT, Register Op0) {
5355 if (RetVT.SimpleTy != MVT::v2f64)
5356 return Register();
5357 if ((Subtarget->isNeonAvailable())) {
5358 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f64, RC: &AArch64::FPR128RegClass, Op0);
5359 }
5360 return Register();
5361}
5362
5363Register fastEmit_ISD_STRICT_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
5364 switch (VT.SimpleTy) {
5365 case MVT::f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(RetVT, Op0);
5366 case MVT::f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(RetVT, Op0);
5367 case MVT::f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(RetVT, Op0);
5368 case MVT::v4f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(RetVT, Op0);
5369 case MVT::v8f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(RetVT, Op0);
5370 case MVT::v2f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(RetVT, Op0);
5371 case MVT::v4f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(RetVT, Op0);
5372 case MVT::v2f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(RetVT, Op0);
5373 default: return Register();
5374 }
5375}
5376
5377// FastEmit functions for ISD::STRICT_LLROUND.
5378
5379Register fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5380 if (RetVT.SimpleTy != MVT::i64)
5381 return Register();
5382 if ((Subtarget->hasFullFP16())) {
5383 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
5384 }
5385 return Register();
5386}
5387
5388Register fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5389 if (RetVT.SimpleTy != MVT::i64)
5390 return Register();
5391 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
5392}
5393
5394Register fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5395 if (RetVT.SimpleTy != MVT::i64)
5396 return Register();
5397 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
5398}
5399
5400Register fastEmit_ISD_STRICT_LLROUND_r(MVT VT, MVT RetVT, Register Op0) {
5401 switch (VT.SimpleTy) {
5402 case MVT::f16: return fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(RetVT, Op0);
5403 case MVT::f32: return fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(RetVT, Op0);
5404 case MVT::f64: return fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(RetVT, Op0);
5405 default: return Register();
5406 }
5407}
5408
5409// FastEmit functions for ISD::STRICT_LROUND.
5410
5411Register fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(Register Op0) {
5412 if ((Subtarget->hasFullFP16())) {
5413 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWHr, RC: &AArch64::GPR32RegClass, Op0);
5414 }
5415 return Register();
5416}
5417
5418Register fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(Register Op0) {
5419 if ((Subtarget->hasFullFP16())) {
5420 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
5421 }
5422 return Register();
5423}
5424
5425Register fastEmit_ISD_STRICT_LROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5426switch (RetVT.SimpleTy) {
5427 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(Op0);
5428 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(Op0);
5429 default: return Register();
5430}
5431}
5432
5433Register fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(Register Op0) {
5434 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWSr, RC: &AArch64::GPR32RegClass, Op0);
5435}
5436
5437Register fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(Register Op0) {
5438 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
5439}
5440
5441Register fastEmit_ISD_STRICT_LROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5442switch (RetVT.SimpleTy) {
5443 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(Op0);
5444 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(Op0);
5445 default: return Register();
5446}
5447}
5448
5449Register fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(Register Op0) {
5450 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWDr, RC: &AArch64::GPR32RegClass, Op0);
5451}
5452
5453Register fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(Register Op0) {
5454 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
5455}
5456
5457Register fastEmit_ISD_STRICT_LROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5458switch (RetVT.SimpleTy) {
5459 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(Op0);
5460 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(Op0);
5461 default: return Register();
5462}
5463}
5464
5465Register fastEmit_ISD_STRICT_LROUND_r(MVT VT, MVT RetVT, Register Op0) {
5466 switch (VT.SimpleTy) {
5467 case MVT::f16: return fastEmit_ISD_STRICT_LROUND_MVT_f16_r(RetVT, Op0);
5468 case MVT::f32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_r(RetVT, Op0);
5469 case MVT::f64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_r(RetVT, Op0);
5470 default: return Register();
5471 }
5472}
5473
5474// FastEmit functions for ISD::STRICT_SINT_TO_FP.
5475
5476Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5477 if ((Subtarget->hasFPRCVT())) {
5478 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5479 }
5480 if ((Subtarget->hasFullFP16())) {
5481 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5482 }
5483 return Register();
5484}
5485
5486Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5487 if ((Subtarget->hasFPARMv8())) {
5488 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5489 }
5490 return Register();
5491}
5492
5493Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5494 if ((Subtarget->hasFPRCVT())) {
5495 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5496 }
5497 if ((Subtarget->hasFPARMv8())) {
5498 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5499 }
5500 return Register();
5501}
5502
5503Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5504switch (RetVT.SimpleTy) {
5505 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5506 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5507 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5508 default: return Register();
5509}
5510}
5511
5512Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5513 if ((Subtarget->hasFPRCVT())) {
5514 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5515 }
5516 if ((Subtarget->hasFullFP16())) {
5517 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5518 }
5519 return Register();
5520}
5521
5522Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5523 if ((Subtarget->hasFPRCVT())) {
5524 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5525 }
5526 if ((Subtarget->hasFPARMv8())) {
5527 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5528 }
5529 return Register();
5530}
5531
5532Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5533 if ((Subtarget->hasFPARMv8())) {
5534 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5535 }
5536 return Register();
5537}
5538
5539Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5540switch (RetVT.SimpleTy) {
5541 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5542 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5543 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5544 default: return Register();
5545}
5546}
5547
5548Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5549 if (RetVT.SimpleTy != MVT::v4f16)
5550 return Register();
5551 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5552 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5553 }
5554 return Register();
5555}
5556
5557Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5558 if (RetVT.SimpleTy != MVT::v8f16)
5559 return Register();
5560 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5561 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5562 }
5563 return Register();
5564}
5565
5566Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5567 if (RetVT.SimpleTy != MVT::v2f32)
5568 return Register();
5569 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5570 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5571 }
5572 return Register();
5573}
5574
5575Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5576 if (RetVT.SimpleTy != MVT::v4f32)
5577 return Register();
5578 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5579 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5580 }
5581 return Register();
5582}
5583
5584Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5585 if (RetVT.SimpleTy != MVT::v2f64)
5586 return Register();
5587 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5588 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5589 }
5590 return Register();
5591}
5592
5593Register fastEmit_ISD_STRICT_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
5594 switch (VT.SimpleTy) {
5595 case MVT::i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
5596 case MVT::i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
5597 case MVT::v4i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
5598 case MVT::v8i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
5599 case MVT::v2i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
5600 case MVT::v4i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
5601 case MVT::v2i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
5602 default: return Register();
5603 }
5604}
5605
5606// FastEmit functions for ISD::STRICT_UINT_TO_FP.
5607
5608Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5609 if ((Subtarget->hasFPRCVT())) {
5610 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5611 }
5612 if ((Subtarget->hasFullFP16())) {
5613 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5614 }
5615 return Register();
5616}
5617
5618Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5619 if ((Subtarget->hasFPARMv8())) {
5620 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5621 }
5622 return Register();
5623}
5624
5625Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5626 if ((Subtarget->hasFPRCVT())) {
5627 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5628 }
5629 if ((Subtarget->hasFPARMv8())) {
5630 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5631 }
5632 return Register();
5633}
5634
5635Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5636switch (RetVT.SimpleTy) {
5637 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5638 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5639 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5640 default: return Register();
5641}
5642}
5643
5644Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5645 if ((Subtarget->hasFPRCVT())) {
5646 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5647 }
5648 if ((Subtarget->hasFullFP16())) {
5649 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5650 }
5651 return Register();
5652}
5653
5654Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5655 if ((Subtarget->hasFPRCVT())) {
5656 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5657 }
5658 if ((Subtarget->hasFPARMv8())) {
5659 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5660 }
5661 return Register();
5662}
5663
5664Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5665 if ((Subtarget->hasFPARMv8())) {
5666 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5667 }
5668 return Register();
5669}
5670
5671Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5672switch (RetVT.SimpleTy) {
5673 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5674 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5675 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5676 default: return Register();
5677}
5678}
5679
5680Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5681 if (RetVT.SimpleTy != MVT::v4f16)
5682 return Register();
5683 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5684 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5685 }
5686 return Register();
5687}
5688
5689Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5690 if (RetVT.SimpleTy != MVT::v8f16)
5691 return Register();
5692 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5693 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5694 }
5695 return Register();
5696}
5697
5698Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5699 if (RetVT.SimpleTy != MVT::v2f32)
5700 return Register();
5701 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5702 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5703 }
5704 return Register();
5705}
5706
5707Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5708 if (RetVT.SimpleTy != MVT::v4f32)
5709 return Register();
5710 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5711 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5712 }
5713 return Register();
5714}
5715
5716Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5717 if (RetVT.SimpleTy != MVT::v2f64)
5718 return Register();
5719 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5720 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5721 }
5722 return Register();
5723}
5724
5725Register fastEmit_ISD_STRICT_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
5726 switch (VT.SimpleTy) {
5727 case MVT::i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(RetVT, Op0);
5728 case MVT::i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(RetVT, Op0);
5729 case MVT::v4i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
5730 case MVT::v8i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
5731 case MVT::v2i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
5732 case MVT::v4i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
5733 case MVT::v2i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
5734 default: return Register();
5735 }
5736}
5737
5738// FastEmit functions for ISD::TRUNCATE.
5739
5740Register fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, Register Op0) {
5741 if (RetVT.SimpleTy != MVT::i32)
5742 return Register();
5743 return fastEmitInst_extractsubreg(RetVT, Op0, Idx: AArch64::sub_32);
5744}
5745
5746Register fastEmit_ISD_TRUNCATE_MVT_v8i16_r(MVT RetVT, Register Op0) {
5747 if (RetVT.SimpleTy != MVT::v8i8)
5748 return Register();
5749 if ((Subtarget->isNeonAvailable())) {
5750 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5751 }
5752 return Register();
5753}
5754
5755Register fastEmit_ISD_TRUNCATE_MVT_v4i32_r(MVT RetVT, Register Op0) {
5756 if (RetVT.SimpleTy != MVT::v4i16)
5757 return Register();
5758 if ((Subtarget->isNeonAvailable())) {
5759 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5760 }
5761 return Register();
5762}
5763
5764Register fastEmit_ISD_TRUNCATE_MVT_v2i64_r(MVT RetVT, Register Op0) {
5765 if (RetVT.SimpleTy != MVT::v2i32)
5766 return Register();
5767 if ((Subtarget->isNeonAvailable())) {
5768 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5769 }
5770 return Register();
5771}
5772
5773Register fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, Register Op0) {
5774 switch (VT.SimpleTy) {
5775 case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0);
5776 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i16_r(RetVT, Op0);
5777 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_MVT_v4i32_r(RetVT, Op0);
5778 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_MVT_v2i64_r(RetVT, Op0);
5779 default: return Register();
5780 }
5781}
5782
5783// FastEmit functions for ISD::TRUNCATE_SSAT_S.
5784
5785Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v8i16_r(MVT RetVT, Register Op0) {
5786 if (RetVT.SimpleTy != MVT::v8i8)
5787 return Register();
5788 if ((Subtarget->isNeonAvailable())) {
5789 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5790 }
5791 return Register();
5792}
5793
5794Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v4i32_r(MVT RetVT, Register Op0) {
5795 if (RetVT.SimpleTy != MVT::v4i16)
5796 return Register();
5797 if ((Subtarget->isNeonAvailable())) {
5798 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5799 }
5800 return Register();
5801}
5802
5803Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v2i64_r(MVT RetVT, Register Op0) {
5804 if (RetVT.SimpleTy != MVT::v2i32)
5805 return Register();
5806 if ((Subtarget->isNeonAvailable())) {
5807 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5808 }
5809 return Register();
5810}
5811
5812Register fastEmit_ISD_TRUNCATE_SSAT_S_r(MVT VT, MVT RetVT, Register Op0) {
5813 switch (VT.SimpleTy) {
5814 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v8i16_r(RetVT, Op0);
5815 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v4i32_r(RetVT, Op0);
5816 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v2i64_r(RetVT, Op0);
5817 default: return Register();
5818 }
5819}
5820
5821// FastEmit functions for ISD::TRUNCATE_SSAT_U.
5822
5823Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v8i16_r(MVT RetVT, Register Op0) {
5824 if (RetVT.SimpleTy != MVT::v8i8)
5825 return Register();
5826 if ((Subtarget->isNeonAvailable())) {
5827 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5828 }
5829 return Register();
5830}
5831
5832Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v4i32_r(MVT RetVT, Register Op0) {
5833 if (RetVT.SimpleTy != MVT::v4i16)
5834 return Register();
5835 if ((Subtarget->isNeonAvailable())) {
5836 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5837 }
5838 return Register();
5839}
5840
5841Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v2i64_r(MVT RetVT, Register Op0) {
5842 if (RetVT.SimpleTy != MVT::v2i32)
5843 return Register();
5844 if ((Subtarget->isNeonAvailable())) {
5845 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5846 }
5847 return Register();
5848}
5849
5850Register fastEmit_ISD_TRUNCATE_SSAT_U_r(MVT VT, MVT RetVT, Register Op0) {
5851 switch (VT.SimpleTy) {
5852 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v8i16_r(RetVT, Op0);
5853 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v4i32_r(RetVT, Op0);
5854 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v2i64_r(RetVT, Op0);
5855 default: return Register();
5856 }
5857}
5858
5859// FastEmit functions for ISD::TRUNCATE_USAT_U.
5860
5861Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v8i16_r(MVT RetVT, Register Op0) {
5862 if (RetVT.SimpleTy != MVT::v8i8)
5863 return Register();
5864 if ((Subtarget->isNeonAvailable())) {
5865 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5866 }
5867 return Register();
5868}
5869
5870Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v4i32_r(MVT RetVT, Register Op0) {
5871 if (RetVT.SimpleTy != MVT::v4i16)
5872 return Register();
5873 if ((Subtarget->isNeonAvailable())) {
5874 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5875 }
5876 return Register();
5877}
5878
5879Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v2i64_r(MVT RetVT, Register Op0) {
5880 if (RetVT.SimpleTy != MVT::v2i32)
5881 return Register();
5882 if ((Subtarget->isNeonAvailable())) {
5883 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5884 }
5885 return Register();
5886}
5887
5888Register fastEmit_ISD_TRUNCATE_USAT_U_r(MVT VT, MVT RetVT, Register Op0) {
5889 switch (VT.SimpleTy) {
5890 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v8i16_r(RetVT, Op0);
5891 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v4i32_r(RetVT, Op0);
5892 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v2i64_r(RetVT, Op0);
5893 default: return Register();
5894 }
5895}
5896
5897// FastEmit functions for ISD::UINT_TO_FP.
5898
5899Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5900 if ((Subtarget->hasFPRCVT())) {
5901 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5902 }
5903 if ((Subtarget->hasFullFP16())) {
5904 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5905 }
5906 return Register();
5907}
5908
5909Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5910 if ((Subtarget->hasFPARMv8())) {
5911 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5912 }
5913 return Register();
5914}
5915
5916Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5917 if ((Subtarget->hasFPRCVT())) {
5918 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5919 }
5920 if ((Subtarget->hasFPARMv8())) {
5921 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5922 }
5923 return Register();
5924}
5925
5926Register fastEmit_ISD_UINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5927switch (RetVT.SimpleTy) {
5928 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5929 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5930 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5931 default: return Register();
5932}
5933}
5934
5935Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5936 if ((Subtarget->hasFPRCVT())) {
5937 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5938 }
5939 if ((Subtarget->hasFullFP16())) {
5940 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5941 }
5942 return Register();
5943}
5944
5945Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5946 if ((Subtarget->hasFPRCVT())) {
5947 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5948 }
5949 if ((Subtarget->hasFPARMv8())) {
5950 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5951 }
5952 return Register();
5953}
5954
5955Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5956 if ((Subtarget->hasFPARMv8())) {
5957 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5958 }
5959 return Register();
5960}
5961
5962Register fastEmit_ISD_UINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5963switch (RetVT.SimpleTy) {
5964 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5965 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5966 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5967 default: return Register();
5968}
5969}
5970
5971Register fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5972 if (RetVT.SimpleTy != MVT::v4f16)
5973 return Register();
5974 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5975 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5976 }
5977 return Register();
5978}
5979
5980Register fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5981 if (RetVT.SimpleTy != MVT::v8f16)
5982 return Register();
5983 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5984 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5985 }
5986 return Register();
5987}
5988
5989Register fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5990 if (RetVT.SimpleTy != MVT::v2f32)
5991 return Register();
5992 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5993 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5994 }
5995 return Register();
5996}
5997
5998Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5999 if (RetVT.SimpleTy != MVT::v4f32)
6000 return Register();
6001 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
6002 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
6003 }
6004 return Register();
6005}
6006
6007Register fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
6008 if (RetVT.SimpleTy != MVT::v2f64)
6009 return Register();
6010 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
6011 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
6012 }
6013 return Register();
6014}
6015
6016Register fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
6017 switch (VT.SimpleTy) {
6018 case MVT::i32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_r(RetVT, Op0);
6019 case MVT::i64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_r(RetVT, Op0);
6020 case MVT::v4i16: return fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
6021 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
6022 case MVT::v2i32: return fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
6023 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
6024 case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
6025 default: return Register();
6026 }
6027}
6028
6029// FastEmit functions for ISD::VECREDUCE_ADD.
6030
6031Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i8_r(MVT RetVT, Register Op0) {
6032 if (RetVT.SimpleTy != MVT::i8)
6033 return Register();
6034 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6035}
6036
6037Register fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(MVT RetVT, Register Op0) {
6038 if (RetVT.SimpleTy != MVT::i8)
6039 return Register();
6040 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6041}
6042
6043Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i16_r(MVT RetVT, Register Op0) {
6044 if (RetVT.SimpleTy != MVT::i16)
6045 return Register();
6046 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6047}
6048
6049Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(MVT RetVT, Register Op0) {
6050 if (RetVT.SimpleTy != MVT::i16)
6051 return Register();
6052 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6053}
6054
6055Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(MVT RetVT, Register Op0) {
6056 if (RetVT.SimpleTy != MVT::i32)
6057 return Register();
6058 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6059}
6060
6061Register fastEmit_ISD_VECREDUCE_ADD_MVT_v2i64_r(MVT RetVT, Register Op0) {
6062 if (RetVT.SimpleTy != MVT::i64)
6063 return Register();
6064 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6065}
6066
6067Register fastEmit_ISD_VECREDUCE_ADD_r(MVT VT, MVT RetVT, Register Op0) {
6068 switch (VT.SimpleTy) {
6069 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i8_r(RetVT, Op0);
6070 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(RetVT, Op0);
6071 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i16_r(RetVT, Op0);
6072 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(RetVT, Op0);
6073 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(RetVT, Op0);
6074 case MVT::v2i64: return fastEmit_ISD_VECREDUCE_ADD_MVT_v2i64_r(RetVT, Op0);
6075 default: return Register();
6076 }
6077}
6078
6079// FastEmit functions for ISD::VECREDUCE_FADD.
6080
6081Register fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(MVT RetVT, Register Op0) {
6082 if (RetVT.SimpleTy != MVT::f32)
6083 return Register();
6084 return fastEmitInst_r(MachineInstOpcode: AArch64::FADDPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6085}
6086
6087Register fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(MVT RetVT, Register Op0) {
6088 if (RetVT.SimpleTy != MVT::f64)
6089 return Register();
6090 return fastEmitInst_r(MachineInstOpcode: AArch64::FADDPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6091}
6092
6093Register fastEmit_ISD_VECREDUCE_FADD_r(MVT VT, MVT RetVT, Register Op0) {
6094 switch (VT.SimpleTy) {
6095 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(RetVT, Op0);
6096 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(RetVT, Op0);
6097 default: return Register();
6098 }
6099}
6100
6101// FastEmit functions for ISD::VECREDUCE_FMAX.
6102
6103Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f16_r(MVT RetVT, Register Op0) {
6104 if (RetVT.SimpleTy != MVT::f16)
6105 return Register();
6106 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6107 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6108 }
6109 return Register();
6110}
6111
6112Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v8f16_r(MVT RetVT, Register Op0) {
6113 if (RetVT.SimpleTy != MVT::f16)
6114 return Register();
6115 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6116 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6117 }
6118 return Register();
6119}
6120
6121Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f32_r(MVT RetVT, Register Op0) {
6122 if (RetVT.SimpleTy != MVT::f32)
6123 return Register();
6124 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6125}
6126
6127Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f32_r(MVT RetVT, Register Op0) {
6128 if (RetVT.SimpleTy != MVT::f32)
6129 return Register();
6130 if ((Subtarget->isNeonAvailable())) {
6131 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6132 }
6133 return Register();
6134}
6135
6136Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f64_r(MVT RetVT, Register Op0) {
6137 if (RetVT.SimpleTy != MVT::f64)
6138 return Register();
6139 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6140}
6141
6142Register fastEmit_ISD_VECREDUCE_FMAX_r(MVT VT, MVT RetVT, Register Op0) {
6143 switch (VT.SimpleTy) {
6144 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f16_r(RetVT, Op0);
6145 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v8f16_r(RetVT, Op0);
6146 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f32_r(RetVT, Op0);
6147 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f32_r(RetVT, Op0);
6148 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f64_r(RetVT, Op0);
6149 default: return Register();
6150 }
6151}
6152
6153// FastEmit functions for ISD::VECREDUCE_FMAXIMUM.
6154
6155Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f16_r(MVT RetVT, Register Op0) {
6156 if (RetVT.SimpleTy != MVT::f16)
6157 return Register();
6158 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6159 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6160 }
6161 return Register();
6162}
6163
6164Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v8f16_r(MVT RetVT, Register Op0) {
6165 if (RetVT.SimpleTy != MVT::f16)
6166 return Register();
6167 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6168 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6169 }
6170 return Register();
6171}
6172
6173Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f32_r(MVT RetVT, Register Op0) {
6174 if (RetVT.SimpleTy != MVT::f32)
6175 return Register();
6176 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6177}
6178
6179Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f32_r(MVT RetVT, Register Op0) {
6180 if (RetVT.SimpleTy != MVT::f32)
6181 return Register();
6182 if ((Subtarget->isNeonAvailable())) {
6183 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6184 }
6185 return Register();
6186}
6187
6188Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f64_r(MVT RetVT, Register Op0) {
6189 if (RetVT.SimpleTy != MVT::f64)
6190 return Register();
6191 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6192}
6193
6194Register fastEmit_ISD_VECREDUCE_FMAXIMUM_r(MVT VT, MVT RetVT, Register Op0) {
6195 switch (VT.SimpleTy) {
6196 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f16_r(RetVT, Op0);
6197 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v8f16_r(RetVT, Op0);
6198 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f32_r(RetVT, Op0);
6199 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f32_r(RetVT, Op0);
6200 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f64_r(RetVT, Op0);
6201 default: return Register();
6202 }
6203}
6204
6205// FastEmit functions for ISD::VECREDUCE_FMIN.
6206
6207Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f16_r(MVT RetVT, Register Op0) {
6208 if (RetVT.SimpleTy != MVT::f16)
6209 return Register();
6210 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6211 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6212 }
6213 return Register();
6214}
6215
6216Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v8f16_r(MVT RetVT, Register Op0) {
6217 if (RetVT.SimpleTy != MVT::f16)
6218 return Register();
6219 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6220 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6221 }
6222 return Register();
6223}
6224
6225Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f32_r(MVT RetVT, Register Op0) {
6226 if (RetVT.SimpleTy != MVT::f32)
6227 return Register();
6228 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6229}
6230
6231Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f32_r(MVT RetVT, Register Op0) {
6232 if (RetVT.SimpleTy != MVT::f32)
6233 return Register();
6234 if ((Subtarget->isNeonAvailable())) {
6235 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6236 }
6237 return Register();
6238}
6239
6240Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f64_r(MVT RetVT, Register Op0) {
6241 if (RetVT.SimpleTy != MVT::f64)
6242 return Register();
6243 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6244}
6245
6246Register fastEmit_ISD_VECREDUCE_FMIN_r(MVT VT, MVT RetVT, Register Op0) {
6247 switch (VT.SimpleTy) {
6248 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f16_r(RetVT, Op0);
6249 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v8f16_r(RetVT, Op0);
6250 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f32_r(RetVT, Op0);
6251 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f32_r(RetVT, Op0);
6252 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f64_r(RetVT, Op0);
6253 default: return Register();
6254 }
6255}
6256
6257// FastEmit functions for ISD::VECREDUCE_FMINIMUM.
6258
6259Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f16_r(MVT RetVT, Register Op0) {
6260 if (RetVT.SimpleTy != MVT::f16)
6261 return Register();
6262 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6263 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6264 }
6265 return Register();
6266}
6267
6268Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v8f16_r(MVT RetVT, Register Op0) {
6269 if (RetVT.SimpleTy != MVT::f16)
6270 return Register();
6271 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6272 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6273 }
6274 return Register();
6275}
6276
6277Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f32_r(MVT RetVT, Register Op0) {
6278 if (RetVT.SimpleTy != MVT::f32)
6279 return Register();
6280 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6281}
6282
6283Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f32_r(MVT RetVT, Register Op0) {
6284 if (RetVT.SimpleTy != MVT::f32)
6285 return Register();
6286 if ((Subtarget->isNeonAvailable())) {
6287 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6288 }
6289 return Register();
6290}
6291
6292Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f64_r(MVT RetVT, Register Op0) {
6293 if (RetVT.SimpleTy != MVT::f64)
6294 return Register();
6295 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6296}
6297
6298Register fastEmit_ISD_VECREDUCE_FMINIMUM_r(MVT VT, MVT RetVT, Register Op0) {
6299 switch (VT.SimpleTy) {
6300 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f16_r(RetVT, Op0);
6301 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v8f16_r(RetVT, Op0);
6302 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f32_r(RetVT, Op0);
6303 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f32_r(RetVT, Op0);
6304 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f64_r(RetVT, Op0);
6305 default: return Register();
6306 }
6307}
6308
6309// FastEmit functions for ISD::VECREDUCE_SMAX.
6310
6311Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i8_r(MVT RetVT, Register Op0) {
6312 if (RetVT.SimpleTy != MVT::i8)
6313 return Register();
6314 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6315}
6316
6317Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v16i8_r(MVT RetVT, Register Op0) {
6318 if (RetVT.SimpleTy != MVT::i8)
6319 return Register();
6320 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6321}
6322
6323Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i16_r(MVT RetVT, Register Op0) {
6324 if (RetVT.SimpleTy != MVT::i16)
6325 return Register();
6326 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6327}
6328
6329Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i16_r(MVT RetVT, Register Op0) {
6330 if (RetVT.SimpleTy != MVT::i16)
6331 return Register();
6332 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6333}
6334
6335Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i32_r(MVT RetVT, Register Op0) {
6336 if (RetVT.SimpleTy != MVT::i32)
6337 return Register();
6338 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6339}
6340
6341Register fastEmit_ISD_VECREDUCE_SMAX_r(MVT VT, MVT RetVT, Register Op0) {
6342 switch (VT.SimpleTy) {
6343 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i8_r(RetVT, Op0);
6344 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v16i8_r(RetVT, Op0);
6345 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i16_r(RetVT, Op0);
6346 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i16_r(RetVT, Op0);
6347 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i32_r(RetVT, Op0);
6348 default: return Register();
6349 }
6350}
6351
6352// FastEmit functions for ISD::VECREDUCE_SMIN.
6353
6354Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i8_r(MVT RetVT, Register Op0) {
6355 if (RetVT.SimpleTy != MVT::i8)
6356 return Register();
6357 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6358}
6359
6360Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v16i8_r(MVT RetVT, Register Op0) {
6361 if (RetVT.SimpleTy != MVT::i8)
6362 return Register();
6363 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6364}
6365
6366Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i16_r(MVT RetVT, Register Op0) {
6367 if (RetVT.SimpleTy != MVT::i16)
6368 return Register();
6369 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6370}
6371
6372Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i16_r(MVT RetVT, Register Op0) {
6373 if (RetVT.SimpleTy != MVT::i16)
6374 return Register();
6375 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6376}
6377
6378Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i32_r(MVT RetVT, Register Op0) {
6379 if (RetVT.SimpleTy != MVT::i32)
6380 return Register();
6381 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6382}
6383
6384Register fastEmit_ISD_VECREDUCE_SMIN_r(MVT VT, MVT RetVT, Register Op0) {
6385 switch (VT.SimpleTy) {
6386 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i8_r(RetVT, Op0);
6387 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v16i8_r(RetVT, Op0);
6388 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i16_r(RetVT, Op0);
6389 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i16_r(RetVT, Op0);
6390 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i32_r(RetVT, Op0);
6391 default: return Register();
6392 }
6393}
6394
6395// FastEmit functions for ISD::VECREDUCE_UMAX.
6396
6397Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i8_r(MVT RetVT, Register Op0) {
6398 if (RetVT.SimpleTy != MVT::i8)
6399 return Register();
6400 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6401}
6402
6403Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v16i8_r(MVT RetVT, Register Op0) {
6404 if (RetVT.SimpleTy != MVT::i8)
6405 return Register();
6406 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6407}
6408
6409Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i16_r(MVT RetVT, Register Op0) {
6410 if (RetVT.SimpleTy != MVT::i16)
6411 return Register();
6412 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6413}
6414
6415Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i16_r(MVT RetVT, Register Op0) {
6416 if (RetVT.SimpleTy != MVT::i16)
6417 return Register();
6418 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6419}
6420
6421Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i32_r(MVT RetVT, Register Op0) {
6422 if (RetVT.SimpleTy != MVT::i32)
6423 return Register();
6424 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6425}
6426
6427Register fastEmit_ISD_VECREDUCE_UMAX_r(MVT VT, MVT RetVT, Register Op0) {
6428 switch (VT.SimpleTy) {
6429 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i8_r(RetVT, Op0);
6430 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v16i8_r(RetVT, Op0);
6431 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i16_r(RetVT, Op0);
6432 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i16_r(RetVT, Op0);
6433 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i32_r(RetVT, Op0);
6434 default: return Register();
6435 }
6436}
6437
6438// FastEmit functions for ISD::VECREDUCE_UMIN.
6439
6440Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i8_r(MVT RetVT, Register Op0) {
6441 if (RetVT.SimpleTy != MVT::i8)
6442 return Register();
6443 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6444}
6445
6446Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v16i8_r(MVT RetVT, Register Op0) {
6447 if (RetVT.SimpleTy != MVT::i8)
6448 return Register();
6449 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6450}
6451
6452Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i16_r(MVT RetVT, Register Op0) {
6453 if (RetVT.SimpleTy != MVT::i16)
6454 return Register();
6455 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6456}
6457
6458Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i16_r(MVT RetVT, Register Op0) {
6459 if (RetVT.SimpleTy != MVT::i16)
6460 return Register();
6461 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6462}
6463
6464Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i32_r(MVT RetVT, Register Op0) {
6465 if (RetVT.SimpleTy != MVT::i32)
6466 return Register();
6467 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6468}
6469
6470Register fastEmit_ISD_VECREDUCE_UMIN_r(MVT VT, MVT RetVT, Register Op0) {
6471 switch (VT.SimpleTy) {
6472 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i8_r(RetVT, Op0);
6473 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v16i8_r(RetVT, Op0);
6474 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i16_r(RetVT, Op0);
6475 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i16_r(RetVT, Op0);
6476 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i32_r(RetVT, Op0);
6477 default: return Register();
6478 }
6479}
6480
6481// FastEmit functions for ISD::VECTOR_REVERSE.
6482
6483Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(MVT RetVT, Register Op0) {
6484 if (RetVT.SimpleTy != MVT::nxv2i1)
6485 return Register();
6486 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6487 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_D, RC: &AArch64::PPRRegClass, Op0);
6488 }
6489 return Register();
6490}
6491
6492Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(MVT RetVT, Register Op0) {
6493 if (RetVT.SimpleTy != MVT::nxv4i1)
6494 return Register();
6495 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6496 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_S, RC: &AArch64::PPRRegClass, Op0);
6497 }
6498 return Register();
6499}
6500
6501Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(MVT RetVT, Register Op0) {
6502 if (RetVT.SimpleTy != MVT::nxv8i1)
6503 return Register();
6504 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6505 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_H, RC: &AArch64::PPRRegClass, Op0);
6506 }
6507 return Register();
6508}
6509
6510Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(MVT RetVT, Register Op0) {
6511 if (RetVT.SimpleTy != MVT::nxv16i1)
6512 return Register();
6513 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6514 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_B, RC: &AArch64::PPRRegClass, Op0);
6515 }
6516 return Register();
6517}
6518
6519Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
6520 if (RetVT.SimpleTy != MVT::nxv16i8)
6521 return Register();
6522 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6523 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_B, RC: &AArch64::ZPRRegClass, Op0);
6524 }
6525 return Register();
6526}
6527
6528Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
6529 if (RetVT.SimpleTy != MVT::nxv8i16)
6530 return Register();
6531 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6532 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6533 }
6534 return Register();
6535}
6536
6537Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
6538 if (RetVT.SimpleTy != MVT::nxv4i32)
6539 return Register();
6540 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6541 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6542 }
6543 return Register();
6544}
6545
6546Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(MVT RetVT, Register Op0) {
6547 if (RetVT.SimpleTy != MVT::nxv2i64)
6548 return Register();
6549 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6550 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6551 }
6552 return Register();
6553}
6554
6555Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(MVT RetVT, Register Op0) {
6556 if (RetVT.SimpleTy != MVT::nxv2f16)
6557 return Register();
6558 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6559 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6560 }
6561 return Register();
6562}
6563
6564Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(MVT RetVT, Register Op0) {
6565 if (RetVT.SimpleTy != MVT::nxv4f16)
6566 return Register();
6567 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6568 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6569 }
6570 return Register();
6571}
6572
6573Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
6574 if (RetVT.SimpleTy != MVT::nxv8f16)
6575 return Register();
6576 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6577 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6578 }
6579 return Register();
6580}
6581
6582Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(MVT RetVT, Register Op0) {
6583 if (RetVT.SimpleTy != MVT::nxv2bf16)
6584 return Register();
6585 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6586 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6587 }
6588 return Register();
6589}
6590
6591Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(MVT RetVT, Register Op0) {
6592 if (RetVT.SimpleTy != MVT::nxv4bf16)
6593 return Register();
6594 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6595 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6596 }
6597 return Register();
6598}
6599
6600Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(MVT RetVT, Register Op0) {
6601 if (RetVT.SimpleTy != MVT::nxv8bf16)
6602 return Register();
6603 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6604 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6605 }
6606 return Register();
6607}
6608
6609Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(MVT RetVT, Register Op0) {
6610 if (RetVT.SimpleTy != MVT::nxv2f32)
6611 return Register();
6612 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6613 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6614 }
6615 return Register();
6616}
6617
6618Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
6619 if (RetVT.SimpleTy != MVT::nxv4f32)
6620 return Register();
6621 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6622 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6623 }
6624 return Register();
6625}
6626
6627Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
6628 if (RetVT.SimpleTy != MVT::nxv2f64)
6629 return Register();
6630 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6631 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6632 }
6633 return Register();
6634}
6635
6636Register fastEmit_ISD_VECTOR_REVERSE_r(MVT VT, MVT RetVT, Register Op0) {
6637 switch (VT.SimpleTy) {
6638 case MVT::nxv2i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(RetVT, Op0);
6639 case MVT::nxv4i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(RetVT, Op0);
6640 case MVT::nxv8i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(RetVT, Op0);
6641 case MVT::nxv16i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(RetVT, Op0);
6642 case MVT::nxv16i8: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(RetVT, Op0);
6643 case MVT::nxv8i16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(RetVT, Op0);
6644 case MVT::nxv4i32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(RetVT, Op0);
6645 case MVT::nxv2i64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(RetVT, Op0);
6646 case MVT::nxv2f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(RetVT, Op0);
6647 case MVT::nxv4f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(RetVT, Op0);
6648 case MVT::nxv8f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(RetVT, Op0);
6649 case MVT::nxv2bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(RetVT, Op0);
6650 case MVT::nxv4bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(RetVT, Op0);
6651 case MVT::nxv8bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(RetVT, Op0);
6652 case MVT::nxv2f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(RetVT, Op0);
6653 case MVT::nxv4f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(RetVT, Op0);
6654 case MVT::nxv2f64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(RetVT, Op0);
6655 default: return Register();
6656 }
6657}
6658
6659// Top-level FastEmit function.
6660
6661Register fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, Register Op0) override {
6662 switch (Opcode) {
6663 case AArch64ISD::COALESCER_BARRIER: return fastEmit_AArch64ISD_COALESCER_BARRIER_r(VT, RetVT, Op0);
6664 case AArch64ISD::DUP: return fastEmit_AArch64ISD_DUP_r(VT, RetVT, Op0);
6665 case AArch64ISD::FCVTXN: return fastEmit_AArch64ISD_FCVTXN_r(VT, RetVT, Op0);
6666 case AArch64ISD::FRECPE: return fastEmit_AArch64ISD_FRECPE_r(VT, RetVT, Op0);
6667 case AArch64ISD::FRSQRTE: return fastEmit_AArch64ISD_FRSQRTE_r(VT, RetVT, Op0);
6668 case AArch64ISD::REV16: return fastEmit_AArch64ISD_REV16_r(VT, RetVT, Op0);
6669 case AArch64ISD::REV32: return fastEmit_AArch64ISD_REV32_r(VT, RetVT, Op0);
6670 case AArch64ISD::REV64: return fastEmit_AArch64ISD_REV64_r(VT, RetVT, Op0);
6671 case AArch64ISD::SADDLP: return fastEmit_AArch64ISD_SADDLP_r(VT, RetVT, Op0);
6672 case AArch64ISD::SITOF: return fastEmit_AArch64ISD_SITOF_r(VT, RetVT, Op0);
6673 case AArch64ISD::SQABS: return fastEmit_AArch64ISD_SQABS_r(VT, RetVT, Op0);
6674 case AArch64ISD::SQNEG: return fastEmit_AArch64ISD_SQNEG_r(VT, RetVT, Op0);
6675 case AArch64ISD::SUNPKHI: return fastEmit_AArch64ISD_SUNPKHI_r(VT, RetVT, Op0);
6676 case AArch64ISD::SUNPKLO: return fastEmit_AArch64ISD_SUNPKLO_r(VT, RetVT, Op0);
6677 case AArch64ISD::UADDLP: return fastEmit_AArch64ISD_UADDLP_r(VT, RetVT, Op0);
6678 case AArch64ISD::UITOF: return fastEmit_AArch64ISD_UITOF_r(VT, RetVT, Op0);
6679 case AArch64ISD::UUNPKHI: return fastEmit_AArch64ISD_UUNPKHI_r(VT, RetVT, Op0);
6680 case AArch64ISD::UUNPKLO: return fastEmit_AArch64ISD_UUNPKLO_r(VT, RetVT, Op0);
6681 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
6682 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
6683 case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0);
6684 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
6685 case ISD::CTLS: return fastEmit_ISD_CTLS_r(VT, RetVT, Op0);
6686 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
6687 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
6688 case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0);
6689 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
6690 case ISD::FCEIL: return fastEmit_ISD_FCEIL_r(VT, RetVT, Op0);
6691 case ISD::FFLOOR: return fastEmit_ISD_FFLOOR_r(VT, RetVT, Op0);
6692 case ISD::FNEARBYINT: return fastEmit_ISD_FNEARBYINT_r(VT, RetVT, Op0);
6693 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
6694 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
6695 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
6696 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
6697 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
6698 case ISD::FRINT: return fastEmit_ISD_FRINT_r(VT, RetVT, Op0);
6699 case ISD::FROUND: return fastEmit_ISD_FROUND_r(VT, RetVT, Op0);
6700 case ISD::FROUNDEVEN: return fastEmit_ISD_FROUNDEVEN_r(VT, RetVT, Op0);
6701 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
6702 case ISD::FTRUNC: return fastEmit_ISD_FTRUNC_r(VT, RetVT, Op0);
6703 case ISD::LLROUND: return fastEmit_ISD_LLROUND_r(VT, RetVT, Op0);
6704 case ISD::LROUND: return fastEmit_ISD_LROUND_r(VT, RetVT, Op0);
6705 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
6706 case ISD::SPLAT_VECTOR: return fastEmit_ISD_SPLAT_VECTOR_r(VT, RetVT, Op0);
6707 case ISD::STRICT_FCEIL: return fastEmit_ISD_STRICT_FCEIL_r(VT, RetVT, Op0);
6708 case ISD::STRICT_FFLOOR: return fastEmit_ISD_STRICT_FFLOOR_r(VT, RetVT, Op0);
6709 case ISD::STRICT_FNEARBYINT: return fastEmit_ISD_STRICT_FNEARBYINT_r(VT, RetVT, Op0);
6710 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0);
6711 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0);
6712 case ISD::STRICT_FP_TO_SINT: return fastEmit_ISD_STRICT_FP_TO_SINT_r(VT, RetVT, Op0);
6713 case ISD::STRICT_FP_TO_UINT: return fastEmit_ISD_STRICT_FP_TO_UINT_r(VT, RetVT, Op0);
6714 case ISD::STRICT_FRINT: return fastEmit_ISD_STRICT_FRINT_r(VT, RetVT, Op0);
6715 case ISD::STRICT_FROUND: return fastEmit_ISD_STRICT_FROUND_r(VT, RetVT, Op0);
6716 case ISD::STRICT_FROUNDEVEN: return fastEmit_ISD_STRICT_FROUNDEVEN_r(VT, RetVT, Op0);
6717 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0);
6718 case ISD::STRICT_FTRUNC: return fastEmit_ISD_STRICT_FTRUNC_r(VT, RetVT, Op0);
6719 case ISD::STRICT_LLROUND: return fastEmit_ISD_STRICT_LLROUND_r(VT, RetVT, Op0);
6720 case ISD::STRICT_LROUND: return fastEmit_ISD_STRICT_LROUND_r(VT, RetVT, Op0);
6721 case ISD::STRICT_SINT_TO_FP: return fastEmit_ISD_STRICT_SINT_TO_FP_r(VT, RetVT, Op0);
6722 case ISD::STRICT_UINT_TO_FP: return fastEmit_ISD_STRICT_UINT_TO_FP_r(VT, RetVT, Op0);
6723 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
6724 case ISD::TRUNCATE_SSAT_S: return fastEmit_ISD_TRUNCATE_SSAT_S_r(VT, RetVT, Op0);
6725 case ISD::TRUNCATE_SSAT_U: return fastEmit_ISD_TRUNCATE_SSAT_U_r(VT, RetVT, Op0);
6726 case ISD::TRUNCATE_USAT_U: return fastEmit_ISD_TRUNCATE_USAT_U_r(VT, RetVT, Op0);
6727 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
6728 case ISD::VECREDUCE_ADD: return fastEmit_ISD_VECREDUCE_ADD_r(VT, RetVT, Op0);
6729 case ISD::VECREDUCE_FADD: return fastEmit_ISD_VECREDUCE_FADD_r(VT, RetVT, Op0);
6730 case ISD::VECREDUCE_FMAX: return fastEmit_ISD_VECREDUCE_FMAX_r(VT, RetVT, Op0);
6731 case ISD::VECREDUCE_FMAXIMUM: return fastEmit_ISD_VECREDUCE_FMAXIMUM_r(VT, RetVT, Op0);
6732 case ISD::VECREDUCE_FMIN: return fastEmit_ISD_VECREDUCE_FMIN_r(VT, RetVT, Op0);
6733 case ISD::VECREDUCE_FMINIMUM: return fastEmit_ISD_VECREDUCE_FMINIMUM_r(VT, RetVT, Op0);
6734 case ISD::VECREDUCE_SMAX: return fastEmit_ISD_VECREDUCE_SMAX_r(VT, RetVT, Op0);
6735 case ISD::VECREDUCE_SMIN: return fastEmit_ISD_VECREDUCE_SMIN_r(VT, RetVT, Op0);
6736 case ISD::VECREDUCE_UMAX: return fastEmit_ISD_VECREDUCE_UMAX_r(VT, RetVT, Op0);
6737 case ISD::VECREDUCE_UMIN: return fastEmit_ISD_VECREDUCE_UMIN_r(VT, RetVT, Op0);
6738 case ISD::VECTOR_REVERSE: return fastEmit_ISD_VECTOR_REVERSE_r(VT, RetVT, Op0);
6739 default: return Register();
6740 }
6741}
6742
6743// FastEmit functions for AArch64ISD::ADDHN.
6744
6745Register fastEmit_AArch64ISD_ADDHN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6746 if (RetVT.SimpleTy != MVT::v8i8)
6747 return Register();
6748 if ((Subtarget->isNeonAvailable())) {
6749 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDHNv8i16_v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
6750 }
6751 return Register();
6752}
6753
6754Register fastEmit_AArch64ISD_ADDHN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6755 if (RetVT.SimpleTy != MVT::v4i16)
6756 return Register();
6757 if ((Subtarget->isNeonAvailable())) {
6758 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDHNv4i32_v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6759 }
6760 return Register();
6761}
6762
6763Register fastEmit_AArch64ISD_ADDHN_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6764 if (RetVT.SimpleTy != MVT::v2i32)
6765 return Register();
6766 if ((Subtarget->isNeonAvailable())) {
6767 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDHNv2i64_v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6768 }
6769 return Register();
6770}
6771
6772Register fastEmit_AArch64ISD_ADDHN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6773 switch (VT.SimpleTy) {
6774 case MVT::v8i16: return fastEmit_AArch64ISD_ADDHN_MVT_v8i16_rr(RetVT, Op0, Op1);
6775 case MVT::v4i32: return fastEmit_AArch64ISD_ADDHN_MVT_v4i32_rr(RetVT, Op0, Op1);
6776 case MVT::v2i64: return fastEmit_AArch64ISD_ADDHN_MVT_v2i64_rr(RetVT, Op0, Op1);
6777 default: return Register();
6778 }
6779}
6780
6781// FastEmit functions for AArch64ISD::ADDP.
6782
6783Register fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6784 if (RetVT.SimpleTy != MVT::v8i8)
6785 return Register();
6786 if ((Subtarget->isNeonAvailable())) {
6787 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
6788 }
6789 return Register();
6790}
6791
6792Register fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6793 if (RetVT.SimpleTy != MVT::v16i8)
6794 return Register();
6795 if ((Subtarget->isNeonAvailable())) {
6796 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
6797 }
6798 return Register();
6799}
6800
6801Register fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6802 if (RetVT.SimpleTy != MVT::v4i16)
6803 return Register();
6804 if ((Subtarget->isNeonAvailable())) {
6805 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6806 }
6807 return Register();
6808}
6809
6810Register fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6811 if (RetVT.SimpleTy != MVT::v8i16)
6812 return Register();
6813 if ((Subtarget->isNeonAvailable())) {
6814 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
6815 }
6816 return Register();
6817}
6818
6819Register fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6820 if (RetVT.SimpleTy != MVT::v2i32)
6821 return Register();
6822 if ((Subtarget->isNeonAvailable())) {
6823 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6824 }
6825 return Register();
6826}
6827
6828Register fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6829 if (RetVT.SimpleTy != MVT::v4i32)
6830 return Register();
6831 if ((Subtarget->isNeonAvailable())) {
6832 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
6833 }
6834 return Register();
6835}
6836
6837Register fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6838 if (RetVT.SimpleTy != MVT::v2i64)
6839 return Register();
6840 if ((Subtarget->isNeonAvailable())) {
6841 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
6842 }
6843 return Register();
6844}
6845
6846Register fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
6847 if (RetVT.SimpleTy != MVT::v4f16)
6848 return Register();
6849 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6850 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6851 }
6852 return Register();
6853}
6854
6855Register fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
6856 if (RetVT.SimpleTy != MVT::v8f16)
6857 return Register();
6858 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6859 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
6860 }
6861 return Register();
6862}
6863
6864Register fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
6865 if (RetVT.SimpleTy != MVT::v2f32)
6866 return Register();
6867 if ((Subtarget->isNeonAvailable())) {
6868 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6869 }
6870 return Register();
6871}
6872
6873Register fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
6874 if (RetVT.SimpleTy != MVT::v4f32)
6875 return Register();
6876 if ((Subtarget->isNeonAvailable())) {
6877 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
6878 }
6879 return Register();
6880}
6881
6882Register fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
6883 if (RetVT.SimpleTy != MVT::v2f64)
6884 return Register();
6885 if ((Subtarget->isNeonAvailable())) {
6886 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
6887 }
6888 return Register();
6889}
6890
6891Register fastEmit_AArch64ISD_ADDP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6892 switch (VT.SimpleTy) {
6893 case MVT::v8i8: return fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(RetVT, Op0, Op1);
6894 case MVT::v16i8: return fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(RetVT, Op0, Op1);
6895 case MVT::v4i16: return fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(RetVT, Op0, Op1);
6896 case MVT::v8i16: return fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(RetVT, Op0, Op1);
6897 case MVT::v2i32: return fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(RetVT, Op0, Op1);
6898 case MVT::v4i32: return fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(RetVT, Op0, Op1);
6899 case MVT::v2i64: return fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(RetVT, Op0, Op1);
6900 case MVT::v4f16: return fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(RetVT, Op0, Op1);
6901 case MVT::v8f16: return fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(RetVT, Op0, Op1);
6902 case MVT::v2f32: return fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(RetVT, Op0, Op1);
6903 case MVT::v4f32: return fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(RetVT, Op0, Op1);
6904 case MVT::v2f64: return fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(RetVT, Op0, Op1);
6905 default: return Register();
6906 }
6907}
6908
6909// FastEmit functions for AArch64ISD::BIC.
6910
6911Register fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6912 if (RetVT.SimpleTy != MVT::nxv16i8)
6913 return Register();
6914 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6915 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6916 }
6917 return Register();
6918}
6919
6920Register fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6921 if (RetVT.SimpleTy != MVT::nxv8i16)
6922 return Register();
6923 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6924 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6925 }
6926 return Register();
6927}
6928
6929Register fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6930 if (RetVT.SimpleTy != MVT::nxv4i32)
6931 return Register();
6932 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6933 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6934 }
6935 return Register();
6936}
6937
6938Register fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6939 if (RetVT.SimpleTy != MVT::nxv2i64)
6940 return Register();
6941 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6942 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6943 }
6944 return Register();
6945}
6946
6947Register fastEmit_AArch64ISD_BIC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6948 switch (VT.SimpleTy) {
6949 case MVT::nxv16i8: return fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(RetVT, Op0, Op1);
6950 case MVT::nxv8i16: return fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(RetVT, Op0, Op1);
6951 case MVT::nxv4i32: return fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(RetVT, Op0, Op1);
6952 case MVT::nxv2i64: return fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(RetVT, Op0, Op1);
6953 default: return Register();
6954 }
6955}
6956
6957// FastEmit functions for AArch64ISD::BRKB_Z.
6958
6959Register fastEmit_AArch64ISD_BRKB_Z_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
6960 if (RetVT.SimpleTy != MVT::nxv16i1)
6961 return Register();
6962 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6963 return fastEmitInst_rr(MachineInstOpcode: AArch64::BRKB_PPzP, RC: &AArch64::PPRRegClass, Op0, Op1);
6964 }
6965 return Register();
6966}
6967
6968Register fastEmit_AArch64ISD_BRKB_Z_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6969 switch (VT.SimpleTy) {
6970 case MVT::nxv16i1: return fastEmit_AArch64ISD_BRKB_Z_MVT_nxv16i1_rr(RetVT, Op0, Op1);
6971 default: return Register();
6972 }
6973}
6974
6975// FastEmit functions for AArch64ISD::CMTST.
6976
6977Register fastEmit_AArch64ISD_CMTST_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6978 if (RetVT.SimpleTy != MVT::v8i8)
6979 return Register();
6980 if ((Subtarget->isNeonAvailable())) {
6981 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
6982 }
6983 return Register();
6984}
6985
6986Register fastEmit_AArch64ISD_CMTST_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6987 if (RetVT.SimpleTy != MVT::v16i8)
6988 return Register();
6989 if ((Subtarget->isNeonAvailable())) {
6990 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
6991 }
6992 return Register();
6993}
6994
6995Register fastEmit_AArch64ISD_CMTST_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6996 if (RetVT.SimpleTy != MVT::v4i16)
6997 return Register();
6998 if ((Subtarget->isNeonAvailable())) {
6999 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7000 }
7001 return Register();
7002}
7003
7004Register fastEmit_AArch64ISD_CMTST_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7005 if (RetVT.SimpleTy != MVT::v8i16)
7006 return Register();
7007 if ((Subtarget->isNeonAvailable())) {
7008 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7009 }
7010 return Register();
7011}
7012
7013Register fastEmit_AArch64ISD_CMTST_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7014 if (RetVT.SimpleTy != MVT::v2i32)
7015 return Register();
7016 if ((Subtarget->isNeonAvailable())) {
7017 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7018 }
7019 return Register();
7020}
7021
7022Register fastEmit_AArch64ISD_CMTST_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7023 if (RetVT.SimpleTy != MVT::v4i32)
7024 return Register();
7025 if ((Subtarget->isNeonAvailable())) {
7026 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7027 }
7028 return Register();
7029}
7030
7031Register fastEmit_AArch64ISD_CMTST_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7032 if (RetVT.SimpleTy != MVT::v1i64)
7033 return Register();
7034 if ((Subtarget->isNeonAvailable())) {
7035 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7036 }
7037 return Register();
7038}
7039
7040Register fastEmit_AArch64ISD_CMTST_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7041 if (RetVT.SimpleTy != MVT::v2i64)
7042 return Register();
7043 if ((Subtarget->isNeonAvailable())) {
7044 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7045 }
7046 return Register();
7047}
7048
7049Register fastEmit_AArch64ISD_CMTST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7050 switch (VT.SimpleTy) {
7051 case MVT::v8i8: return fastEmit_AArch64ISD_CMTST_MVT_v8i8_rr(RetVT, Op0, Op1);
7052 case MVT::v16i8: return fastEmit_AArch64ISD_CMTST_MVT_v16i8_rr(RetVT, Op0, Op1);
7053 case MVT::v4i16: return fastEmit_AArch64ISD_CMTST_MVT_v4i16_rr(RetVT, Op0, Op1);
7054 case MVT::v8i16: return fastEmit_AArch64ISD_CMTST_MVT_v8i16_rr(RetVT, Op0, Op1);
7055 case MVT::v2i32: return fastEmit_AArch64ISD_CMTST_MVT_v2i32_rr(RetVT, Op0, Op1);
7056 case MVT::v4i32: return fastEmit_AArch64ISD_CMTST_MVT_v4i32_rr(RetVT, Op0, Op1);
7057 case MVT::v1i64: return fastEmit_AArch64ISD_CMTST_MVT_v1i64_rr(RetVT, Op0, Op1);
7058 case MVT::v2i64: return fastEmit_AArch64ISD_CMTST_MVT_v2i64_rr(RetVT, Op0, Op1);
7059 default: return Register();
7060 }
7061}
7062
7063// FastEmit functions for AArch64ISD::CTTZ_ELTS_ZERO_POISON.
7064
7065Register fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
7066 if (RetVT.SimpleTy != MVT::i64)
7067 return Register();
7068 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()))) {
7069 return fastEmitInst_rr(MachineInstOpcode: AArch64::FIRSTP_XPP_D, RC: &AArch64::GPR64RegClass, Op0, Op1);
7070 }
7071 return Register();
7072}
7073
7074Register fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
7075 if (RetVT.SimpleTy != MVT::i64)
7076 return Register();
7077 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()))) {
7078 return fastEmitInst_rr(MachineInstOpcode: AArch64::FIRSTP_XPP_S, RC: &AArch64::GPR64RegClass, Op0, Op1);
7079 }
7080 return Register();
7081}
7082
7083Register fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
7084 if (RetVT.SimpleTy != MVT::i64)
7085 return Register();
7086 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()))) {
7087 return fastEmitInst_rr(MachineInstOpcode: AArch64::FIRSTP_XPP_H, RC: &AArch64::GPR64RegClass, Op0, Op1);
7088 }
7089 return Register();
7090}
7091
7092Register fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7093 if (RetVT.SimpleTy != MVT::i64)
7094 return Register();
7095 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()))) {
7096 return fastEmitInst_rr(MachineInstOpcode: AArch64::FIRSTP_XPP_B, RC: &AArch64::GPR64RegClass, Op0, Op1);
7097 }
7098 return Register();
7099}
7100
7101Register fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7102 switch (VT.SimpleTy) {
7103 case MVT::nxv2i1: return fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv2i1_rr(RetVT, Op0, Op1);
7104 case MVT::nxv4i1: return fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv4i1_rr(RetVT, Op0, Op1);
7105 case MVT::nxv8i1: return fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv8i1_rr(RetVT, Op0, Op1);
7106 case MVT::nxv16i1: return fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7107 default: return Register();
7108 }
7109}
7110
7111// FastEmit functions for AArch64ISD::FCMEQ.
7112
7113Register fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7114 if (RetVT.SimpleTy != MVT::i32)
7115 return Register();
7116 if ((Subtarget->isNeonAvailable())) {
7117 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7118 }
7119 return Register();
7120}
7121
7122Register fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7123 if (RetVT.SimpleTy != MVT::i64)
7124 return Register();
7125 if ((Subtarget->isNeonAvailable())) {
7126 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7127 }
7128 return Register();
7129}
7130
7131Register fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7132 if (RetVT.SimpleTy != MVT::v4i16)
7133 return Register();
7134 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7135 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7136 }
7137 return Register();
7138}
7139
7140Register fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7141 if (RetVT.SimpleTy != MVT::v8i16)
7142 return Register();
7143 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7144 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7145 }
7146 return Register();
7147}
7148
7149Register fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7150 if (RetVT.SimpleTy != MVT::v2i32)
7151 return Register();
7152 if ((Subtarget->isNeonAvailable())) {
7153 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7154 }
7155 return Register();
7156}
7157
7158Register fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7159 if (RetVT.SimpleTy != MVT::v4i32)
7160 return Register();
7161 if ((Subtarget->isNeonAvailable())) {
7162 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7163 }
7164 return Register();
7165}
7166
7167Register fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7168 if (RetVT.SimpleTy != MVT::v1i64)
7169 return Register();
7170 if ((Subtarget->isNeonAvailable())) {
7171 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7172 }
7173 return Register();
7174}
7175
7176Register fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7177 if (RetVT.SimpleTy != MVT::v2i64)
7178 return Register();
7179 if ((Subtarget->isNeonAvailable())) {
7180 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7181 }
7182 return Register();
7183}
7184
7185Register fastEmit_AArch64ISD_FCMEQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7186 switch (VT.SimpleTy) {
7187 case MVT::f32: return fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(RetVT, Op0, Op1);
7188 case MVT::f64: return fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(RetVT, Op0, Op1);
7189 case MVT::v4f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(RetVT, Op0, Op1);
7190 case MVT::v8f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(RetVT, Op0, Op1);
7191 case MVT::v2f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(RetVT, Op0, Op1);
7192 case MVT::v4f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(RetVT, Op0, Op1);
7193 case MVT::v1f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(RetVT, Op0, Op1);
7194 case MVT::v2f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(RetVT, Op0, Op1);
7195 default: return Register();
7196 }
7197}
7198
7199// FastEmit functions for AArch64ISD::FCMGE.
7200
7201Register fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7202 if (RetVT.SimpleTy != MVT::i32)
7203 return Register();
7204 if ((Subtarget->isNeonAvailable())) {
7205 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7206 }
7207 return Register();
7208}
7209
7210Register fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7211 if (RetVT.SimpleTy != MVT::i64)
7212 return Register();
7213 if ((Subtarget->isNeonAvailable())) {
7214 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7215 }
7216 return Register();
7217}
7218
7219Register fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7220 if (RetVT.SimpleTy != MVT::v4i16)
7221 return Register();
7222 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7223 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7224 }
7225 return Register();
7226}
7227
7228Register fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7229 if (RetVT.SimpleTy != MVT::v8i16)
7230 return Register();
7231 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7232 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7233 }
7234 return Register();
7235}
7236
7237Register fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7238 if (RetVT.SimpleTy != MVT::v2i32)
7239 return Register();
7240 if ((Subtarget->isNeonAvailable())) {
7241 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7242 }
7243 return Register();
7244}
7245
7246Register fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7247 if (RetVT.SimpleTy != MVT::v4i32)
7248 return Register();
7249 if ((Subtarget->isNeonAvailable())) {
7250 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7251 }
7252 return Register();
7253}
7254
7255Register fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7256 if (RetVT.SimpleTy != MVT::v1i64)
7257 return Register();
7258 if ((Subtarget->isNeonAvailable())) {
7259 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7260 }
7261 return Register();
7262}
7263
7264Register fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7265 if (RetVT.SimpleTy != MVT::v2i64)
7266 return Register();
7267 if ((Subtarget->isNeonAvailable())) {
7268 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7269 }
7270 return Register();
7271}
7272
7273Register fastEmit_AArch64ISD_FCMGE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7274 switch (VT.SimpleTy) {
7275 case MVT::f32: return fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(RetVT, Op0, Op1);
7276 case MVT::f64: return fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(RetVT, Op0, Op1);
7277 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(RetVT, Op0, Op1);
7278 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(RetVT, Op0, Op1);
7279 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(RetVT, Op0, Op1);
7280 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(RetVT, Op0, Op1);
7281 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(RetVT, Op0, Op1);
7282 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(RetVT, Op0, Op1);
7283 default: return Register();
7284 }
7285}
7286
7287// FastEmit functions for AArch64ISD::FCMGT.
7288
7289Register fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7290 if (RetVT.SimpleTy != MVT::i32)
7291 return Register();
7292 if ((Subtarget->isNeonAvailable())) {
7293 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7294 }
7295 return Register();
7296}
7297
7298Register fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7299 if (RetVT.SimpleTy != MVT::i64)
7300 return Register();
7301 if ((Subtarget->isNeonAvailable())) {
7302 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7303 }
7304 return Register();
7305}
7306
7307Register fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7308 if (RetVT.SimpleTy != MVT::v4i16)
7309 return Register();
7310 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7311 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7312 }
7313 return Register();
7314}
7315
7316Register fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7317 if (RetVT.SimpleTy != MVT::v8i16)
7318 return Register();
7319 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7320 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7321 }
7322 return Register();
7323}
7324
7325Register fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7326 if (RetVT.SimpleTy != MVT::v2i32)
7327 return Register();
7328 if ((Subtarget->isNeonAvailable())) {
7329 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7330 }
7331 return Register();
7332}
7333
7334Register fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7335 if (RetVT.SimpleTy != MVT::v4i32)
7336 return Register();
7337 if ((Subtarget->isNeonAvailable())) {
7338 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7339 }
7340 return Register();
7341}
7342
7343Register fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7344 if (RetVT.SimpleTy != MVT::v1i64)
7345 return Register();
7346 if ((Subtarget->isNeonAvailable())) {
7347 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7348 }
7349 return Register();
7350}
7351
7352Register fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7353 if (RetVT.SimpleTy != MVT::v2i64)
7354 return Register();
7355 if ((Subtarget->isNeonAvailable())) {
7356 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7357 }
7358 return Register();
7359}
7360
7361Register fastEmit_AArch64ISD_FCMGT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7362 switch (VT.SimpleTy) {
7363 case MVT::f32: return fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(RetVT, Op0, Op1);
7364 case MVT::f64: return fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(RetVT, Op0, Op1);
7365 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(RetVT, Op0, Op1);
7366 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(RetVT, Op0, Op1);
7367 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(RetVT, Op0, Op1);
7368 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(RetVT, Op0, Op1);
7369 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(RetVT, Op0, Op1);
7370 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(RetVT, Op0, Op1);
7371 default: return Register();
7372 }
7373}
7374
7375// FastEmit functions for AArch64ISD::FRECPS.
7376
7377Register fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7378 if (RetVT.SimpleTy != MVT::f32)
7379 return Register();
7380 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7381}
7382
7383Register fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7384 if (RetVT.SimpleTy != MVT::f64)
7385 return Register();
7386 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7387}
7388
7389Register fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7390 if (RetVT.SimpleTy != MVT::v2f32)
7391 return Register();
7392 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7393}
7394
7395Register fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7396 if (RetVT.SimpleTy != MVT::v4f32)
7397 return Register();
7398 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7399}
7400
7401Register fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7402 if (RetVT.SimpleTy != MVT::v2f64)
7403 return Register();
7404 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7405}
7406
7407Register fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7408 if (RetVT.SimpleTy != MVT::nxv8f16)
7409 return Register();
7410 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7411 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7412 }
7413 return Register();
7414}
7415
7416Register fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7417 if (RetVT.SimpleTy != MVT::nxv4f32)
7418 return Register();
7419 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7420 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7421 }
7422 return Register();
7423}
7424
7425Register fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7426 if (RetVT.SimpleTy != MVT::nxv2f64)
7427 return Register();
7428 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7429 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7430 }
7431 return Register();
7432}
7433
7434Register fastEmit_AArch64ISD_FRECPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7435 switch (VT.SimpleTy) {
7436 case MVT::f32: return fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(RetVT, Op0, Op1);
7437 case MVT::f64: return fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(RetVT, Op0, Op1);
7438 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(RetVT, Op0, Op1);
7439 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(RetVT, Op0, Op1);
7440 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(RetVT, Op0, Op1);
7441 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(RetVT, Op0, Op1);
7442 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(RetVT, Op0, Op1);
7443 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(RetVT, Op0, Op1);
7444 default: return Register();
7445 }
7446}
7447
7448// FastEmit functions for AArch64ISD::FRSQRTS.
7449
7450Register fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7451 if (RetVT.SimpleTy != MVT::f32)
7452 return Register();
7453 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7454}
7455
7456Register fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7457 if (RetVT.SimpleTy != MVT::f64)
7458 return Register();
7459 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7460}
7461
7462Register fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7463 if (RetVT.SimpleTy != MVT::v2f32)
7464 return Register();
7465 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7466}
7467
7468Register fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7469 if (RetVT.SimpleTy != MVT::v4f32)
7470 return Register();
7471 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7472}
7473
7474Register fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7475 if (RetVT.SimpleTy != MVT::v2f64)
7476 return Register();
7477 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7478}
7479
7480Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7481 if (RetVT.SimpleTy != MVT::nxv8f16)
7482 return Register();
7483 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7484 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7485 }
7486 return Register();
7487}
7488
7489Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7490 if (RetVT.SimpleTy != MVT::nxv4f32)
7491 return Register();
7492 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7493 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7494 }
7495 return Register();
7496}
7497
7498Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7499 if (RetVT.SimpleTy != MVT::nxv2f64)
7500 return Register();
7501 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7502 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7503 }
7504 return Register();
7505}
7506
7507Register fastEmit_AArch64ISD_FRSQRTS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7508 switch (VT.SimpleTy) {
7509 case MVT::f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(RetVT, Op0, Op1);
7510 case MVT::f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(RetVT, Op0, Op1);
7511 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(RetVT, Op0, Op1);
7512 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(RetVT, Op0, Op1);
7513 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(RetVT, Op0, Op1);
7514 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(RetVT, Op0, Op1);
7515 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(RetVT, Op0, Op1);
7516 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(RetVT, Op0, Op1);
7517 default: return Register();
7518 }
7519}
7520
7521// FastEmit functions for AArch64ISD::PMULL.
7522
7523Register fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7524 if (RetVT.SimpleTy != MVT::v8i16)
7525 return Register();
7526 if ((Subtarget->isNeonAvailable())) {
7527 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULLv8i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
7528 }
7529 return Register();
7530}
7531
7532Register fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7533 if (RetVT.SimpleTy != MVT::v16i8)
7534 return Register();
7535 if ((Subtarget->hasAES())) {
7536 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULLv1i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7537 }
7538 return Register();
7539}
7540
7541Register fastEmit_AArch64ISD_PMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7542 switch (VT.SimpleTy) {
7543 case MVT::v8i8: return fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
7544 case MVT::v1i64: return fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(RetVT, Op0, Op1);
7545 default: return Register();
7546 }
7547}
7548
7549// FastEmit functions for AArch64ISD::SMULL.
7550
7551Register fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7552 if (RetVT.SimpleTy != MVT::v8i16)
7553 return Register();
7554 if ((Subtarget->isNeonAvailable())) {
7555 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv8i8_v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7556 }
7557 return Register();
7558}
7559
7560Register fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7561 if (RetVT.SimpleTy != MVT::v4i32)
7562 return Register();
7563 if ((Subtarget->isNeonAvailable())) {
7564 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv4i16_v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7565 }
7566 return Register();
7567}
7568
7569Register fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7570 if (RetVT.SimpleTy != MVT::v2i64)
7571 return Register();
7572 if ((Subtarget->isNeonAvailable())) {
7573 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv2i32_v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7574 }
7575 return Register();
7576}
7577
7578Register fastEmit_AArch64ISD_SMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7579 switch (VT.SimpleTy) {
7580 case MVT::v8i8: return fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
7581 case MVT::v4i16: return fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(RetVT, Op0, Op1);
7582 case MVT::v2i32: return fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(RetVT, Op0, Op1);
7583 default: return Register();
7584 }
7585}
7586
7587// FastEmit functions for AArch64ISD::SQADD.
7588
7589Register fastEmit_AArch64ISD_SQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7590 if (RetVT.SimpleTy != MVT::f32)
7591 return Register();
7592 if ((Subtarget->isNeonAvailable())) {
7593 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7594 }
7595 return Register();
7596}
7597
7598Register fastEmit_AArch64ISD_SQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7599 if (RetVT.SimpleTy != MVT::f64)
7600 return Register();
7601 if ((Subtarget->isNeonAvailable())) {
7602 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7603 }
7604 return Register();
7605}
7606
7607Register fastEmit_AArch64ISD_SQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7608 switch (VT.SimpleTy) {
7609 case MVT::f32: return fastEmit_AArch64ISD_SQADD_MVT_f32_rr(RetVT, Op0, Op1);
7610 case MVT::f64: return fastEmit_AArch64ISD_SQADD_MVT_f64_rr(RetVT, Op0, Op1);
7611 default: return Register();
7612 }
7613}
7614
7615// FastEmit functions for AArch64ISD::SQDMULH.
7616
7617Register fastEmit_AArch64ISD_SQDMULH_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7618 if (RetVT.SimpleTy != MVT::f32)
7619 return Register();
7620 if ((Subtarget->isNeonAvailable())) {
7621 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7622 }
7623 return Register();
7624}
7625
7626Register fastEmit_AArch64ISD_SQDMULH_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7627 if (RetVT.SimpleTy != MVT::v4i16)
7628 return Register();
7629 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7630}
7631
7632Register fastEmit_AArch64ISD_SQDMULH_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7633 if (RetVT.SimpleTy != MVT::v8i16)
7634 return Register();
7635 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7636}
7637
7638Register fastEmit_AArch64ISD_SQDMULH_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7639 if (RetVT.SimpleTy != MVT::v2i32)
7640 return Register();
7641 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7642}
7643
7644Register fastEmit_AArch64ISD_SQDMULH_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7645 if (RetVT.SimpleTy != MVT::v4i32)
7646 return Register();
7647 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7648}
7649
7650Register fastEmit_AArch64ISD_SQDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7651 switch (VT.SimpleTy) {
7652 case MVT::f32: return fastEmit_AArch64ISD_SQDMULH_MVT_f32_rr(RetVT, Op0, Op1);
7653 case MVT::v4i16: return fastEmit_AArch64ISD_SQDMULH_MVT_v4i16_rr(RetVT, Op0, Op1);
7654 case MVT::v8i16: return fastEmit_AArch64ISD_SQDMULH_MVT_v8i16_rr(RetVT, Op0, Op1);
7655 case MVT::v2i32: return fastEmit_AArch64ISD_SQDMULH_MVT_v2i32_rr(RetVT, Op0, Op1);
7656 case MVT::v4i32: return fastEmit_AArch64ISD_SQDMULH_MVT_v4i32_rr(RetVT, Op0, Op1);
7657 default: return Register();
7658 }
7659}
7660
7661// FastEmit functions for AArch64ISD::SQDMULL.
7662
7663Register fastEmit_AArch64ISD_SQDMULL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7664 if (RetVT.SimpleTy != MVT::f64)
7665 return Register();
7666 if ((Subtarget->isNeonAvailable())) {
7667 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULLi32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7668 }
7669 return Register();
7670}
7671
7672Register fastEmit_AArch64ISD_SQDMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7673 switch (VT.SimpleTy) {
7674 case MVT::f32: return fastEmit_AArch64ISD_SQDMULL_MVT_f32_rr(RetVT, Op0, Op1);
7675 default: return Register();
7676 }
7677}
7678
7679// FastEmit functions for AArch64ISD::SQRDMULH.
7680
7681Register fastEmit_AArch64ISD_SQRDMULH_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7682 if (RetVT.SimpleTy != MVT::f32)
7683 return Register();
7684 if ((Subtarget->isNeonAvailable())) {
7685 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRDMULHv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7686 }
7687 return Register();
7688}
7689
7690Register fastEmit_AArch64ISD_SQRDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7691 switch (VT.SimpleTy) {
7692 case MVT::f32: return fastEmit_AArch64ISD_SQRDMULH_MVT_f32_rr(RetVT, Op0, Op1);
7693 default: return Register();
7694 }
7695}
7696
7697// FastEmit functions for AArch64ISD::SQRSHL.
7698
7699Register fastEmit_AArch64ISD_SQRSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7700 if (RetVT.SimpleTy != MVT::f32)
7701 return Register();
7702 if ((Subtarget->isNeonAvailable())) {
7703 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7704 }
7705 return Register();
7706}
7707
7708Register fastEmit_AArch64ISD_SQRSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7709 if (RetVT.SimpleTy != MVT::f64)
7710 return Register();
7711 if ((Subtarget->isNeonAvailable())) {
7712 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7713 }
7714 return Register();
7715}
7716
7717Register fastEmit_AArch64ISD_SQRSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7718 switch (VT.SimpleTy) {
7719 case MVT::f32: return fastEmit_AArch64ISD_SQRSHL_MVT_f32_rr(RetVT, Op0, Op1);
7720 case MVT::f64: return fastEmit_AArch64ISD_SQRSHL_MVT_f64_rr(RetVT, Op0, Op1);
7721 default: return Register();
7722 }
7723}
7724
7725// FastEmit functions for AArch64ISD::SQSHL.
7726
7727Register fastEmit_AArch64ISD_SQSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7728 if (RetVT.SimpleTy != MVT::f32)
7729 return Register();
7730 if ((Subtarget->isNeonAvailable())) {
7731 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7732 }
7733 return Register();
7734}
7735
7736Register fastEmit_AArch64ISD_SQSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7737 if (RetVT.SimpleTy != MVT::f64)
7738 return Register();
7739 if ((Subtarget->isNeonAvailable())) {
7740 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7741 }
7742 return Register();
7743}
7744
7745Register fastEmit_AArch64ISD_SQSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7746 switch (VT.SimpleTy) {
7747 case MVT::f32: return fastEmit_AArch64ISD_SQSHL_MVT_f32_rr(RetVT, Op0, Op1);
7748 case MVT::f64: return fastEmit_AArch64ISD_SQSHL_MVT_f64_rr(RetVT, Op0, Op1);
7749 default: return Register();
7750 }
7751}
7752
7753// FastEmit functions for AArch64ISD::SQSUB.
7754
7755Register fastEmit_AArch64ISD_SQSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7756 if (RetVT.SimpleTy != MVT::f32)
7757 return Register();
7758 if ((Subtarget->isNeonAvailable())) {
7759 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7760 }
7761 return Register();
7762}
7763
7764Register fastEmit_AArch64ISD_SQSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7765 if (RetVT.SimpleTy != MVT::f64)
7766 return Register();
7767 if ((Subtarget->isNeonAvailable())) {
7768 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7769 }
7770 return Register();
7771}
7772
7773Register fastEmit_AArch64ISD_SQSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7774 switch (VT.SimpleTy) {
7775 case MVT::f32: return fastEmit_AArch64ISD_SQSUB_MVT_f32_rr(RetVT, Op0, Op1);
7776 case MVT::f64: return fastEmit_AArch64ISD_SQSUB_MVT_f64_rr(RetVT, Op0, Op1);
7777 default: return Register();
7778 }
7779}
7780
7781// FastEmit functions for AArch64ISD::SUQADD.
7782
7783Register fastEmit_AArch64ISD_SUQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7784 if (RetVT.SimpleTy != MVT::f32)
7785 return Register();
7786 if ((Subtarget->isNeonAvailable())) {
7787 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7788 }
7789 return Register();
7790}
7791
7792Register fastEmit_AArch64ISD_SUQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7793 if (RetVT.SimpleTy != MVT::f64)
7794 return Register();
7795 if ((Subtarget->isNeonAvailable())) {
7796 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7797 }
7798 return Register();
7799}
7800
7801Register fastEmit_AArch64ISD_SUQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7802 switch (VT.SimpleTy) {
7803 case MVT::f32: return fastEmit_AArch64ISD_SUQADD_MVT_f32_rr(RetVT, Op0, Op1);
7804 case MVT::f64: return fastEmit_AArch64ISD_SUQADD_MVT_f64_rr(RetVT, Op0, Op1);
7805 default: return Register();
7806 }
7807}
7808
7809// FastEmit functions for AArch64ISD::TBL.
7810
7811Register fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7812 if (RetVT.SimpleTy != MVT::nxv16i8)
7813 return Register();
7814 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7815 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
7816 }
7817 return Register();
7818}
7819
7820Register fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7821 if (RetVT.SimpleTy != MVT::nxv8i16)
7822 return Register();
7823 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7824 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7825 }
7826 return Register();
7827}
7828
7829Register fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7830 if (RetVT.SimpleTy != MVT::nxv4i32)
7831 return Register();
7832 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7833 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7834 }
7835 return Register();
7836}
7837
7838Register fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7839 if (RetVT.SimpleTy != MVT::nxv2i64)
7840 return Register();
7841 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7842 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7843 }
7844 return Register();
7845}
7846
7847Register fastEmit_AArch64ISD_TBL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7848 switch (VT.SimpleTy) {
7849 case MVT::nxv16i8: return fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(RetVT, Op0, Op1);
7850 case MVT::nxv8i16: return fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(RetVT, Op0, Op1);
7851 case MVT::nxv4i32: return fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(RetVT, Op0, Op1);
7852 case MVT::nxv2i64: return fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(RetVT, Op0, Op1);
7853 default: return Register();
7854 }
7855}
7856
7857// FastEmit functions for AArch64ISD::TRN1.
7858
7859Register fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7860 if (RetVT.SimpleTy != MVT::v8i8)
7861 return Register();
7862 if ((Subtarget->isNeonAvailable())) {
7863 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
7864 }
7865 return Register();
7866}
7867
7868Register fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7869 if (RetVT.SimpleTy != MVT::v16i8)
7870 return Register();
7871 if ((Subtarget->isNeonAvailable())) {
7872 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
7873 }
7874 return Register();
7875}
7876
7877Register fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7878 if (RetVT.SimpleTy != MVT::v4i16)
7879 return Register();
7880 if ((Subtarget->isNeonAvailable())) {
7881 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7882 }
7883 return Register();
7884}
7885
7886Register fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7887 if (RetVT.SimpleTy != MVT::v8i16)
7888 return Register();
7889 if ((Subtarget->isNeonAvailable())) {
7890 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7891 }
7892 return Register();
7893}
7894
7895Register fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7896 if (RetVT.SimpleTy != MVT::v2i32)
7897 return Register();
7898 if ((Subtarget->isNeonAvailable())) {
7899 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7900 }
7901 return Register();
7902}
7903
7904Register fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7905 if (RetVT.SimpleTy != MVT::v4i32)
7906 return Register();
7907 if ((Subtarget->isNeonAvailable())) {
7908 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7909 }
7910 return Register();
7911}
7912
7913Register fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7914 if (RetVT.SimpleTy != MVT::v2i64)
7915 return Register();
7916 if ((Subtarget->isNeonAvailable())) {
7917 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7918 }
7919 return Register();
7920}
7921
7922Register fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7923 if (RetVT.SimpleTy != MVT::v4f16)
7924 return Register();
7925 if ((Subtarget->isNeonAvailable())) {
7926 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7927 }
7928 return Register();
7929}
7930
7931Register fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7932 if (RetVT.SimpleTy != MVT::v8f16)
7933 return Register();
7934 if ((Subtarget->isNeonAvailable())) {
7935 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7936 }
7937 return Register();
7938}
7939
7940Register fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
7941 if (RetVT.SimpleTy != MVT::v4bf16)
7942 return Register();
7943 if ((Subtarget->isNeonAvailable())) {
7944 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7945 }
7946 return Register();
7947}
7948
7949Register fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
7950 if (RetVT.SimpleTy != MVT::v8bf16)
7951 return Register();
7952 if ((Subtarget->isNeonAvailable())) {
7953 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7954 }
7955 return Register();
7956}
7957
7958Register fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7959 if (RetVT.SimpleTy != MVT::v2f32)
7960 return Register();
7961 if ((Subtarget->isNeonAvailable())) {
7962 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7963 }
7964 return Register();
7965}
7966
7967Register fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7968 if (RetVT.SimpleTy != MVT::v4f32)
7969 return Register();
7970 if ((Subtarget->isNeonAvailable())) {
7971 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7972 }
7973 return Register();
7974}
7975
7976Register fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7977 if (RetVT.SimpleTy != MVT::v2f64)
7978 return Register();
7979 if ((Subtarget->isNeonAvailable())) {
7980 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7981 }
7982 return Register();
7983}
7984
7985Register fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
7986 if (RetVT.SimpleTy != MVT::nxv2i1)
7987 return Register();
7988 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7989 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
7990 }
7991 return Register();
7992}
7993
7994Register fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
7995 if (RetVT.SimpleTy != MVT::nxv4i1)
7996 return Register();
7997 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7998 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
7999 }
8000 return Register();
8001}
8002
8003Register fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8004 if (RetVT.SimpleTy != MVT::nxv8i1)
8005 return Register();
8006 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8007 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8008 }
8009 return Register();
8010}
8011
8012Register fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8013 if (RetVT.SimpleTy != MVT::nxv16i1)
8014 return Register();
8015 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8016 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8017 }
8018 return Register();
8019}
8020
8021Register fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8022 if (RetVT.SimpleTy != MVT::nxv16i8)
8023 return Register();
8024 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8025 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8026 }
8027 return Register();
8028}
8029
8030Register fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8031 if (RetVT.SimpleTy != MVT::nxv8i16)
8032 return Register();
8033 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8034 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8035 }
8036 return Register();
8037}
8038
8039Register fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8040 if (RetVT.SimpleTy != MVT::nxv4i32)
8041 return Register();
8042 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8043 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8044 }
8045 return Register();
8046}
8047
8048Register fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8049 if (RetVT.SimpleTy != MVT::nxv2i64)
8050 return Register();
8051 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8052 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8053 }
8054 return Register();
8055}
8056
8057Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8058 if (RetVT.SimpleTy != MVT::nxv2f16)
8059 return Register();
8060 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8061 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8062 }
8063 return Register();
8064}
8065
8066Register fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8067 if (RetVT.SimpleTy != MVT::nxv4f16)
8068 return Register();
8069 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8070 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8071 }
8072 return Register();
8073}
8074
8075Register fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8076 if (RetVT.SimpleTy != MVT::nxv8f16)
8077 return Register();
8078 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8079 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8080 }
8081 return Register();
8082}
8083
8084Register fastEmit_AArch64ISD_TRN1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8085 if (RetVT.SimpleTy != MVT::nxv2bf16)
8086 return Register();
8087 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8088 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8089 }
8090 return Register();
8091}
8092
8093Register fastEmit_AArch64ISD_TRN1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8094 if (RetVT.SimpleTy != MVT::nxv4bf16)
8095 return Register();
8096 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8097 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8098 }
8099 return Register();
8100}
8101
8102Register fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8103 if (RetVT.SimpleTy != MVT::nxv8bf16)
8104 return Register();
8105 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8106 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8107 }
8108 return Register();
8109}
8110
8111Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8112 if (RetVT.SimpleTy != MVT::nxv2f32)
8113 return Register();
8114 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8115 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8116 }
8117 return Register();
8118}
8119
8120Register fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8121 if (RetVT.SimpleTy != MVT::nxv4f32)
8122 return Register();
8123 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8124 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8125 }
8126 return Register();
8127}
8128
8129Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8130 if (RetVT.SimpleTy != MVT::nxv2f64)
8131 return Register();
8132 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8133 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8134 }
8135 return Register();
8136}
8137
8138Register fastEmit_AArch64ISD_TRN1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8139 switch (VT.SimpleTy) {
8140 case MVT::v8i8: return fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(RetVT, Op0, Op1);
8141 case MVT::v16i8: return fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(RetVT, Op0, Op1);
8142 case MVT::v4i16: return fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(RetVT, Op0, Op1);
8143 case MVT::v8i16: return fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(RetVT, Op0, Op1);
8144 case MVT::v2i32: return fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(RetVT, Op0, Op1);
8145 case MVT::v4i32: return fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(RetVT, Op0, Op1);
8146 case MVT::v2i64: return fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(RetVT, Op0, Op1);
8147 case MVT::v4f16: return fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(RetVT, Op0, Op1);
8148 case MVT::v8f16: return fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(RetVT, Op0, Op1);
8149 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(RetVT, Op0, Op1);
8150 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(RetVT, Op0, Op1);
8151 case MVT::v2f32: return fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(RetVT, Op0, Op1);
8152 case MVT::v4f32: return fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(RetVT, Op0, Op1);
8153 case MVT::v2f64: return fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(RetVT, Op0, Op1);
8154 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8155 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8156 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8157 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8158 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8159 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8160 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8161 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8162 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8163 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8164 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8165 case MVT::nxv2bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8166 case MVT::nxv4bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8167 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8168 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8169 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8170 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8171 default: return Register();
8172 }
8173}
8174
8175// FastEmit functions for AArch64ISD::TRN2.
8176
8177Register fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8178 if (RetVT.SimpleTy != MVT::v8i8)
8179 return Register();
8180 if ((Subtarget->isNeonAvailable())) {
8181 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8182 }
8183 return Register();
8184}
8185
8186Register fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8187 if (RetVT.SimpleTy != MVT::v16i8)
8188 return Register();
8189 if ((Subtarget->isNeonAvailable())) {
8190 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8191 }
8192 return Register();
8193}
8194
8195Register fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8196 if (RetVT.SimpleTy != MVT::v4i16)
8197 return Register();
8198 if ((Subtarget->isNeonAvailable())) {
8199 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8200 }
8201 return Register();
8202}
8203
8204Register fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8205 if (RetVT.SimpleTy != MVT::v8i16)
8206 return Register();
8207 if ((Subtarget->isNeonAvailable())) {
8208 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8209 }
8210 return Register();
8211}
8212
8213Register fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8214 if (RetVT.SimpleTy != MVT::v2i32)
8215 return Register();
8216 if ((Subtarget->isNeonAvailable())) {
8217 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8218 }
8219 return Register();
8220}
8221
8222Register fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8223 if (RetVT.SimpleTy != MVT::v4i32)
8224 return Register();
8225 if ((Subtarget->isNeonAvailable())) {
8226 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8227 }
8228 return Register();
8229}
8230
8231Register fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8232 if (RetVT.SimpleTy != MVT::v2i64)
8233 return Register();
8234 if ((Subtarget->isNeonAvailable())) {
8235 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8236 }
8237 return Register();
8238}
8239
8240Register fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8241 if (RetVT.SimpleTy != MVT::v4f16)
8242 return Register();
8243 if ((Subtarget->isNeonAvailable())) {
8244 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8245 }
8246 return Register();
8247}
8248
8249Register fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8250 if (RetVT.SimpleTy != MVT::v8f16)
8251 return Register();
8252 if ((Subtarget->isNeonAvailable())) {
8253 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8254 }
8255 return Register();
8256}
8257
8258Register fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8259 if (RetVT.SimpleTy != MVT::v4bf16)
8260 return Register();
8261 if ((Subtarget->isNeonAvailable())) {
8262 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8263 }
8264 return Register();
8265}
8266
8267Register fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8268 if (RetVT.SimpleTy != MVT::v8bf16)
8269 return Register();
8270 if ((Subtarget->isNeonAvailable())) {
8271 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8272 }
8273 return Register();
8274}
8275
8276Register fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8277 if (RetVT.SimpleTy != MVT::v2f32)
8278 return Register();
8279 if ((Subtarget->isNeonAvailable())) {
8280 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8281 }
8282 return Register();
8283}
8284
8285Register fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8286 if (RetVT.SimpleTy != MVT::v4f32)
8287 return Register();
8288 if ((Subtarget->isNeonAvailable())) {
8289 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8290 }
8291 return Register();
8292}
8293
8294Register fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8295 if (RetVT.SimpleTy != MVT::v2f64)
8296 return Register();
8297 if ((Subtarget->isNeonAvailable())) {
8298 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8299 }
8300 return Register();
8301}
8302
8303Register fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8304 if (RetVT.SimpleTy != MVT::nxv2i1)
8305 return Register();
8306 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8307 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8308 }
8309 return Register();
8310}
8311
8312Register fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8313 if (RetVT.SimpleTy != MVT::nxv4i1)
8314 return Register();
8315 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8316 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8317 }
8318 return Register();
8319}
8320
8321Register fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8322 if (RetVT.SimpleTy != MVT::nxv8i1)
8323 return Register();
8324 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8325 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8326 }
8327 return Register();
8328}
8329
8330Register fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8331 if (RetVT.SimpleTy != MVT::nxv16i1)
8332 return Register();
8333 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8334 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8335 }
8336 return Register();
8337}
8338
8339Register fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8340 if (RetVT.SimpleTy != MVT::nxv16i8)
8341 return Register();
8342 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8343 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8344 }
8345 return Register();
8346}
8347
8348Register fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8349 if (RetVT.SimpleTy != MVT::nxv8i16)
8350 return Register();
8351 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8352 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8353 }
8354 return Register();
8355}
8356
8357Register fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8358 if (RetVT.SimpleTy != MVT::nxv4i32)
8359 return Register();
8360 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8361 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8362 }
8363 return Register();
8364}
8365
8366Register fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8367 if (RetVT.SimpleTy != MVT::nxv2i64)
8368 return Register();
8369 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8370 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8371 }
8372 return Register();
8373}
8374
8375Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8376 if (RetVT.SimpleTy != MVT::nxv2f16)
8377 return Register();
8378 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8379 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8380 }
8381 return Register();
8382}
8383
8384Register fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8385 if (RetVT.SimpleTy != MVT::nxv4f16)
8386 return Register();
8387 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8388 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8389 }
8390 return Register();
8391}
8392
8393Register fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8394 if (RetVT.SimpleTy != MVT::nxv8f16)
8395 return Register();
8396 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8397 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8398 }
8399 return Register();
8400}
8401
8402Register fastEmit_AArch64ISD_TRN2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8403 if (RetVT.SimpleTy != MVT::nxv2bf16)
8404 return Register();
8405 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8406 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8407 }
8408 return Register();
8409}
8410
8411Register fastEmit_AArch64ISD_TRN2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8412 if (RetVT.SimpleTy != MVT::nxv4bf16)
8413 return Register();
8414 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8415 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8416 }
8417 return Register();
8418}
8419
8420Register fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8421 if (RetVT.SimpleTy != MVT::nxv8bf16)
8422 return Register();
8423 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8424 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8425 }
8426 return Register();
8427}
8428
8429Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8430 if (RetVT.SimpleTy != MVT::nxv2f32)
8431 return Register();
8432 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8433 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8434 }
8435 return Register();
8436}
8437
8438Register fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8439 if (RetVT.SimpleTy != MVT::nxv4f32)
8440 return Register();
8441 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8442 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8443 }
8444 return Register();
8445}
8446
8447Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8448 if (RetVT.SimpleTy != MVT::nxv2f64)
8449 return Register();
8450 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8451 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8452 }
8453 return Register();
8454}
8455
8456Register fastEmit_AArch64ISD_TRN2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8457 switch (VT.SimpleTy) {
8458 case MVT::v8i8: return fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(RetVT, Op0, Op1);
8459 case MVT::v16i8: return fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(RetVT, Op0, Op1);
8460 case MVT::v4i16: return fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(RetVT, Op0, Op1);
8461 case MVT::v8i16: return fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(RetVT, Op0, Op1);
8462 case MVT::v2i32: return fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(RetVT, Op0, Op1);
8463 case MVT::v4i32: return fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(RetVT, Op0, Op1);
8464 case MVT::v2i64: return fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(RetVT, Op0, Op1);
8465 case MVT::v4f16: return fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(RetVT, Op0, Op1);
8466 case MVT::v8f16: return fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(RetVT, Op0, Op1);
8467 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(RetVT, Op0, Op1);
8468 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(RetVT, Op0, Op1);
8469 case MVT::v2f32: return fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(RetVT, Op0, Op1);
8470 case MVT::v4f32: return fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(RetVT, Op0, Op1);
8471 case MVT::v2f64: return fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(RetVT, Op0, Op1);
8472 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8473 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8474 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8475 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8476 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8477 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8478 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8479 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8480 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8481 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8482 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8483 case MVT::nxv2bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8484 case MVT::nxv4bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8485 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8486 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8487 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8488 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8489 default: return Register();
8490 }
8491}
8492
8493// FastEmit functions for AArch64ISD::UMULL.
8494
8495Register fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8496 if (RetVT.SimpleTy != MVT::v8i16)
8497 return Register();
8498 if ((Subtarget->isNeonAvailable())) {
8499 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv8i8_v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8500 }
8501 return Register();
8502}
8503
8504Register fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8505 if (RetVT.SimpleTy != MVT::v4i32)
8506 return Register();
8507 if ((Subtarget->isNeonAvailable())) {
8508 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv4i16_v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8509 }
8510 return Register();
8511}
8512
8513Register fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8514 if (RetVT.SimpleTy != MVT::v2i64)
8515 return Register();
8516 if ((Subtarget->isNeonAvailable())) {
8517 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv2i32_v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8518 }
8519 return Register();
8520}
8521
8522Register fastEmit_AArch64ISD_UMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8523 switch (VT.SimpleTy) {
8524 case MVT::v8i8: return fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
8525 case MVT::v4i16: return fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(RetVT, Op0, Op1);
8526 case MVT::v2i32: return fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(RetVT, Op0, Op1);
8527 default: return Register();
8528 }
8529}
8530
8531// FastEmit functions for AArch64ISD::UQADD.
8532
8533Register fastEmit_AArch64ISD_UQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8534 if (RetVT.SimpleTy != MVT::f32)
8535 return Register();
8536 if ((Subtarget->isNeonAvailable())) {
8537 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8538 }
8539 return Register();
8540}
8541
8542Register fastEmit_AArch64ISD_UQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8543 if (RetVT.SimpleTy != MVT::f64)
8544 return Register();
8545 if ((Subtarget->isNeonAvailable())) {
8546 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8547 }
8548 return Register();
8549}
8550
8551Register fastEmit_AArch64ISD_UQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8552 switch (VT.SimpleTy) {
8553 case MVT::f32: return fastEmit_AArch64ISD_UQADD_MVT_f32_rr(RetVT, Op0, Op1);
8554 case MVT::f64: return fastEmit_AArch64ISD_UQADD_MVT_f64_rr(RetVT, Op0, Op1);
8555 default: return Register();
8556 }
8557}
8558
8559// FastEmit functions for AArch64ISD::UQRSHL.
8560
8561Register fastEmit_AArch64ISD_UQRSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8562 if (RetVT.SimpleTy != MVT::f32)
8563 return Register();
8564 if ((Subtarget->isNeonAvailable())) {
8565 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQRSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8566 }
8567 return Register();
8568}
8569
8570Register fastEmit_AArch64ISD_UQRSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8571 if (RetVT.SimpleTy != MVT::f64)
8572 return Register();
8573 if ((Subtarget->isNeonAvailable())) {
8574 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQRSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8575 }
8576 return Register();
8577}
8578
8579Register fastEmit_AArch64ISD_UQRSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8580 switch (VT.SimpleTy) {
8581 case MVT::f32: return fastEmit_AArch64ISD_UQRSHL_MVT_f32_rr(RetVT, Op0, Op1);
8582 case MVT::f64: return fastEmit_AArch64ISD_UQRSHL_MVT_f64_rr(RetVT, Op0, Op1);
8583 default: return Register();
8584 }
8585}
8586
8587// FastEmit functions for AArch64ISD::UQSHL.
8588
8589Register fastEmit_AArch64ISD_UQSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8590 if (RetVT.SimpleTy != MVT::f32)
8591 return Register();
8592 if ((Subtarget->isNeonAvailable())) {
8593 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8594 }
8595 return Register();
8596}
8597
8598Register fastEmit_AArch64ISD_UQSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8599 if (RetVT.SimpleTy != MVT::f64)
8600 return Register();
8601 if ((Subtarget->isNeonAvailable())) {
8602 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8603 }
8604 return Register();
8605}
8606
8607Register fastEmit_AArch64ISD_UQSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8608 switch (VT.SimpleTy) {
8609 case MVT::f32: return fastEmit_AArch64ISD_UQSHL_MVT_f32_rr(RetVT, Op0, Op1);
8610 case MVT::f64: return fastEmit_AArch64ISD_UQSHL_MVT_f64_rr(RetVT, Op0, Op1);
8611 default: return Register();
8612 }
8613}
8614
8615// FastEmit functions for AArch64ISD::UQSUB.
8616
8617Register fastEmit_AArch64ISD_UQSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8618 if (RetVT.SimpleTy != MVT::f32)
8619 return Register();
8620 if ((Subtarget->isNeonAvailable())) {
8621 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8622 }
8623 return Register();
8624}
8625
8626Register fastEmit_AArch64ISD_UQSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8627 if (RetVT.SimpleTy != MVT::f64)
8628 return Register();
8629 if ((Subtarget->isNeonAvailable())) {
8630 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8631 }
8632 return Register();
8633}
8634
8635Register fastEmit_AArch64ISD_UQSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8636 switch (VT.SimpleTy) {
8637 case MVT::f32: return fastEmit_AArch64ISD_UQSUB_MVT_f32_rr(RetVT, Op0, Op1);
8638 case MVT::f64: return fastEmit_AArch64ISD_UQSUB_MVT_f64_rr(RetVT, Op0, Op1);
8639 default: return Register();
8640 }
8641}
8642
8643// FastEmit functions for AArch64ISD::USQADD.
8644
8645Register fastEmit_AArch64ISD_USQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8646 if (RetVT.SimpleTy != MVT::f32)
8647 return Register();
8648 if ((Subtarget->isNeonAvailable())) {
8649 return fastEmitInst_rr(MachineInstOpcode: AArch64::USQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8650 }
8651 return Register();
8652}
8653
8654Register fastEmit_AArch64ISD_USQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8655 if (RetVT.SimpleTy != MVT::f64)
8656 return Register();
8657 if ((Subtarget->isNeonAvailable())) {
8658 return fastEmitInst_rr(MachineInstOpcode: AArch64::USQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8659 }
8660 return Register();
8661}
8662
8663Register fastEmit_AArch64ISD_USQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8664 switch (VT.SimpleTy) {
8665 case MVT::f32: return fastEmit_AArch64ISD_USQADD_MVT_f32_rr(RetVT, Op0, Op1);
8666 case MVT::f64: return fastEmit_AArch64ISD_USQADD_MVT_f64_rr(RetVT, Op0, Op1);
8667 default: return Register();
8668 }
8669}
8670
8671// FastEmit functions for AArch64ISD::UZP1.
8672
8673Register fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8674 if (RetVT.SimpleTy != MVT::v8i8)
8675 return Register();
8676 if ((Subtarget->isNeonAvailable())) {
8677 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8678 }
8679 return Register();
8680}
8681
8682Register fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8683 if (RetVT.SimpleTy != MVT::v16i8)
8684 return Register();
8685 if ((Subtarget->isNeonAvailable())) {
8686 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8687 }
8688 return Register();
8689}
8690
8691Register fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8692 if (RetVT.SimpleTy != MVT::v4i16)
8693 return Register();
8694 if ((Subtarget->isNeonAvailable())) {
8695 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8696 }
8697 return Register();
8698}
8699
8700Register fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8701 if (RetVT.SimpleTy != MVT::v8i16)
8702 return Register();
8703 if ((Subtarget->isNeonAvailable())) {
8704 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8705 }
8706 return Register();
8707}
8708
8709Register fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8710 if (RetVT.SimpleTy != MVT::v2i32)
8711 return Register();
8712 if ((Subtarget->isNeonAvailable())) {
8713 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8714 }
8715 return Register();
8716}
8717
8718Register fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8719 if (RetVT.SimpleTy != MVT::v4i32)
8720 return Register();
8721 if ((Subtarget->isNeonAvailable())) {
8722 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8723 }
8724 return Register();
8725}
8726
8727Register fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8728 if (RetVT.SimpleTy != MVT::v2i64)
8729 return Register();
8730 if ((Subtarget->isNeonAvailable())) {
8731 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8732 }
8733 return Register();
8734}
8735
8736Register fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8737 if (RetVT.SimpleTy != MVT::v4f16)
8738 return Register();
8739 if ((Subtarget->isNeonAvailable())) {
8740 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8741 }
8742 return Register();
8743}
8744
8745Register fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8746 if (RetVT.SimpleTy != MVT::v8f16)
8747 return Register();
8748 if ((Subtarget->isNeonAvailable())) {
8749 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8750 }
8751 return Register();
8752}
8753
8754Register fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8755 if (RetVT.SimpleTy != MVT::v4bf16)
8756 return Register();
8757 if ((Subtarget->isNeonAvailable())) {
8758 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8759 }
8760 return Register();
8761}
8762
8763Register fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8764 if (RetVT.SimpleTy != MVT::v8bf16)
8765 return Register();
8766 if ((Subtarget->isNeonAvailable())) {
8767 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8768 }
8769 return Register();
8770}
8771
8772Register fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8773 if (RetVT.SimpleTy != MVT::v2f32)
8774 return Register();
8775 if ((Subtarget->isNeonAvailable())) {
8776 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8777 }
8778 return Register();
8779}
8780
8781Register fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8782 if (RetVT.SimpleTy != MVT::v4f32)
8783 return Register();
8784 if ((Subtarget->isNeonAvailable())) {
8785 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8786 }
8787 return Register();
8788}
8789
8790Register fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8791 if (RetVT.SimpleTy != MVT::v2f64)
8792 return Register();
8793 if ((Subtarget->isNeonAvailable())) {
8794 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8795 }
8796 return Register();
8797}
8798
8799Register fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8800 if (RetVT.SimpleTy != MVT::nxv2i1)
8801 return Register();
8802 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8803 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8804 }
8805 return Register();
8806}
8807
8808Register fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8809 if (RetVT.SimpleTy != MVT::nxv4i1)
8810 return Register();
8811 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8812 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8813 }
8814 return Register();
8815}
8816
8817Register fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8818 if (RetVT.SimpleTy != MVT::nxv8i1)
8819 return Register();
8820 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8821 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8822 }
8823 return Register();
8824}
8825
8826Register fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8827 if (RetVT.SimpleTy != MVT::nxv16i1)
8828 return Register();
8829 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8830 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8831 }
8832 return Register();
8833}
8834
8835Register fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8836 if (RetVT.SimpleTy != MVT::nxv16i8)
8837 return Register();
8838 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8839 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8840 }
8841 return Register();
8842}
8843
8844Register fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8845 if (RetVT.SimpleTy != MVT::nxv8i16)
8846 return Register();
8847 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8848 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8849 }
8850 return Register();
8851}
8852
8853Register fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8854 if (RetVT.SimpleTy != MVT::nxv4i32)
8855 return Register();
8856 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8857 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8858 }
8859 return Register();
8860}
8861
8862Register fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8863 if (RetVT.SimpleTy != MVT::nxv2i64)
8864 return Register();
8865 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8866 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8867 }
8868 return Register();
8869}
8870
8871Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8872 if (RetVT.SimpleTy != MVT::nxv2f16)
8873 return Register();
8874 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8875 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8876 }
8877 return Register();
8878}
8879
8880Register fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8881 if (RetVT.SimpleTy != MVT::nxv4f16)
8882 return Register();
8883 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8884 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8885 }
8886 return Register();
8887}
8888
8889Register fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8890 if (RetVT.SimpleTy != MVT::nxv8f16)
8891 return Register();
8892 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8893 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8894 }
8895 return Register();
8896}
8897
8898Register fastEmit_AArch64ISD_UZP1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8899 if (RetVT.SimpleTy != MVT::nxv2bf16)
8900 return Register();
8901 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8902 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8903 }
8904 return Register();
8905}
8906
8907Register fastEmit_AArch64ISD_UZP1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8908 if (RetVT.SimpleTy != MVT::nxv4bf16)
8909 return Register();
8910 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8911 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8912 }
8913 return Register();
8914}
8915
8916Register fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8917 if (RetVT.SimpleTy != MVT::nxv8bf16)
8918 return Register();
8919 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8920 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8921 }
8922 return Register();
8923}
8924
8925Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8926 if (RetVT.SimpleTy != MVT::nxv2f32)
8927 return Register();
8928 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8929 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8930 }
8931 return Register();
8932}
8933
8934Register fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8935 if (RetVT.SimpleTy != MVT::nxv4f32)
8936 return Register();
8937 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8938 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8939 }
8940 return Register();
8941}
8942
8943Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8944 if (RetVT.SimpleTy != MVT::nxv2f64)
8945 return Register();
8946 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8947 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8948 }
8949 return Register();
8950}
8951
8952Register fastEmit_AArch64ISD_UZP1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8953 switch (VT.SimpleTy) {
8954 case MVT::v8i8: return fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(RetVT, Op0, Op1);
8955 case MVT::v16i8: return fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(RetVT, Op0, Op1);
8956 case MVT::v4i16: return fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(RetVT, Op0, Op1);
8957 case MVT::v8i16: return fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(RetVT, Op0, Op1);
8958 case MVT::v2i32: return fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(RetVT, Op0, Op1);
8959 case MVT::v4i32: return fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(RetVT, Op0, Op1);
8960 case MVT::v2i64: return fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(RetVT, Op0, Op1);
8961 case MVT::v4f16: return fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(RetVT, Op0, Op1);
8962 case MVT::v8f16: return fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(RetVT, Op0, Op1);
8963 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(RetVT, Op0, Op1);
8964 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(RetVT, Op0, Op1);
8965 case MVT::v2f32: return fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(RetVT, Op0, Op1);
8966 case MVT::v4f32: return fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(RetVT, Op0, Op1);
8967 case MVT::v2f64: return fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(RetVT, Op0, Op1);
8968 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8969 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8970 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8971 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8972 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8973 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8974 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8975 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8976 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8977 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8978 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8979 case MVT::nxv2bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8980 case MVT::nxv4bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8981 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8982 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8983 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8984 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8985 default: return Register();
8986 }
8987}
8988
8989// FastEmit functions for AArch64ISD::UZP2.
8990
8991Register fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8992 if (RetVT.SimpleTy != MVT::v8i8)
8993 return Register();
8994 if ((Subtarget->isNeonAvailable())) {
8995 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8996 }
8997 return Register();
8998}
8999
9000Register fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9001 if (RetVT.SimpleTy != MVT::v16i8)
9002 return Register();
9003 if ((Subtarget->isNeonAvailable())) {
9004 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9005 }
9006 return Register();
9007}
9008
9009Register fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9010 if (RetVT.SimpleTy != MVT::v4i16)
9011 return Register();
9012 if ((Subtarget->isNeonAvailable())) {
9013 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9014 }
9015 return Register();
9016}
9017
9018Register fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9019 if (RetVT.SimpleTy != MVT::v8i16)
9020 return Register();
9021 if ((Subtarget->isNeonAvailable())) {
9022 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9023 }
9024 return Register();
9025}
9026
9027Register fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9028 if (RetVT.SimpleTy != MVT::v2i32)
9029 return Register();
9030 if ((Subtarget->isNeonAvailable())) {
9031 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9032 }
9033 return Register();
9034}
9035
9036Register fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9037 if (RetVT.SimpleTy != MVT::v4i32)
9038 return Register();
9039 if ((Subtarget->isNeonAvailable())) {
9040 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9041 }
9042 return Register();
9043}
9044
9045Register fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9046 if (RetVT.SimpleTy != MVT::v2i64)
9047 return Register();
9048 if ((Subtarget->isNeonAvailable())) {
9049 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9050 }
9051 return Register();
9052}
9053
9054Register fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9055 if (RetVT.SimpleTy != MVT::v4f16)
9056 return Register();
9057 if ((Subtarget->isNeonAvailable())) {
9058 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9059 }
9060 return Register();
9061}
9062
9063Register fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9064 if (RetVT.SimpleTy != MVT::v8f16)
9065 return Register();
9066 if ((Subtarget->isNeonAvailable())) {
9067 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9068 }
9069 return Register();
9070}
9071
9072Register fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9073 if (RetVT.SimpleTy != MVT::v4bf16)
9074 return Register();
9075 if ((Subtarget->isNeonAvailable())) {
9076 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9077 }
9078 return Register();
9079}
9080
9081Register fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9082 if (RetVT.SimpleTy != MVT::v8bf16)
9083 return Register();
9084 if ((Subtarget->isNeonAvailable())) {
9085 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9086 }
9087 return Register();
9088}
9089
9090Register fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9091 if (RetVT.SimpleTy != MVT::v2f32)
9092 return Register();
9093 if ((Subtarget->isNeonAvailable())) {
9094 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9095 }
9096 return Register();
9097}
9098
9099Register fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9100 if (RetVT.SimpleTy != MVT::v4f32)
9101 return Register();
9102 if ((Subtarget->isNeonAvailable())) {
9103 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9104 }
9105 return Register();
9106}
9107
9108Register fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9109 if (RetVT.SimpleTy != MVT::v2f64)
9110 return Register();
9111 if ((Subtarget->isNeonAvailable())) {
9112 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9113 }
9114 return Register();
9115}
9116
9117Register fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9118 if (RetVT.SimpleTy != MVT::nxv2i1)
9119 return Register();
9120 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9121 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9122 }
9123 return Register();
9124}
9125
9126Register fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9127 if (RetVT.SimpleTy != MVT::nxv4i1)
9128 return Register();
9129 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9130 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9131 }
9132 return Register();
9133}
9134
9135Register fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9136 if (RetVT.SimpleTy != MVT::nxv8i1)
9137 return Register();
9138 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9139 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9140 }
9141 return Register();
9142}
9143
9144Register fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9145 if (RetVT.SimpleTy != MVT::nxv16i1)
9146 return Register();
9147 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9148 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9149 }
9150 return Register();
9151}
9152
9153Register fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9154 if (RetVT.SimpleTy != MVT::nxv16i8)
9155 return Register();
9156 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9157 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9158 }
9159 return Register();
9160}
9161
9162Register fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9163 if (RetVT.SimpleTy != MVT::nxv8i16)
9164 return Register();
9165 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9166 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9167 }
9168 return Register();
9169}
9170
9171Register fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9172 if (RetVT.SimpleTy != MVT::nxv4i32)
9173 return Register();
9174 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9175 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9176 }
9177 return Register();
9178}
9179
9180Register fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9181 if (RetVT.SimpleTy != MVT::nxv2i64)
9182 return Register();
9183 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9184 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9185 }
9186 return Register();
9187}
9188
9189Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9190 if (RetVT.SimpleTy != MVT::nxv2f16)
9191 return Register();
9192 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9193 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9194 }
9195 return Register();
9196}
9197
9198Register fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9199 if (RetVT.SimpleTy != MVT::nxv4f16)
9200 return Register();
9201 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9202 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9203 }
9204 return Register();
9205}
9206
9207Register fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9208 if (RetVT.SimpleTy != MVT::nxv8f16)
9209 return Register();
9210 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9211 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9212 }
9213 return Register();
9214}
9215
9216Register fastEmit_AArch64ISD_UZP2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9217 if (RetVT.SimpleTy != MVT::nxv2bf16)
9218 return Register();
9219 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9220 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9221 }
9222 return Register();
9223}
9224
9225Register fastEmit_AArch64ISD_UZP2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9226 if (RetVT.SimpleTy != MVT::nxv4bf16)
9227 return Register();
9228 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9229 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9230 }
9231 return Register();
9232}
9233
9234Register fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9235 if (RetVT.SimpleTy != MVT::nxv8bf16)
9236 return Register();
9237 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9238 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9239 }
9240 return Register();
9241}
9242
9243Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9244 if (RetVT.SimpleTy != MVT::nxv2f32)
9245 return Register();
9246 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9247 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9248 }
9249 return Register();
9250}
9251
9252Register fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9253 if (RetVT.SimpleTy != MVT::nxv4f32)
9254 return Register();
9255 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9256 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9257 }
9258 return Register();
9259}
9260
9261Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9262 if (RetVT.SimpleTy != MVT::nxv2f64)
9263 return Register();
9264 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9265 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9266 }
9267 return Register();
9268}
9269
9270Register fastEmit_AArch64ISD_UZP2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9271 switch (VT.SimpleTy) {
9272 case MVT::v8i8: return fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(RetVT, Op0, Op1);
9273 case MVT::v16i8: return fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(RetVT, Op0, Op1);
9274 case MVT::v4i16: return fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(RetVT, Op0, Op1);
9275 case MVT::v8i16: return fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(RetVT, Op0, Op1);
9276 case MVT::v2i32: return fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(RetVT, Op0, Op1);
9277 case MVT::v4i32: return fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(RetVT, Op0, Op1);
9278 case MVT::v2i64: return fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(RetVT, Op0, Op1);
9279 case MVT::v4f16: return fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(RetVT, Op0, Op1);
9280 case MVT::v8f16: return fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(RetVT, Op0, Op1);
9281 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(RetVT, Op0, Op1);
9282 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(RetVT, Op0, Op1);
9283 case MVT::v2f32: return fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(RetVT, Op0, Op1);
9284 case MVT::v4f32: return fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(RetVT, Op0, Op1);
9285 case MVT::v2f64: return fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(RetVT, Op0, Op1);
9286 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9287 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9288 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9289 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9290 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9291 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9292 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9293 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9294 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9295 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9296 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9297 case MVT::nxv2bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9298 case MVT::nxv4bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9299 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9300 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9301 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9302 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9303 default: return Register();
9304 }
9305}
9306
9307// FastEmit functions for AArch64ISD::ZIP1.
9308
9309Register fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9310 if (RetVT.SimpleTy != MVT::v8i8)
9311 return Register();
9312 if ((Subtarget->isNeonAvailable())) {
9313 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9314 }
9315 return Register();
9316}
9317
9318Register fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9319 if (RetVT.SimpleTy != MVT::v16i8)
9320 return Register();
9321 if ((Subtarget->isNeonAvailable())) {
9322 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9323 }
9324 return Register();
9325}
9326
9327Register fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9328 if (RetVT.SimpleTy != MVT::v4i16)
9329 return Register();
9330 if ((Subtarget->isNeonAvailable())) {
9331 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9332 }
9333 return Register();
9334}
9335
9336Register fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9337 if (RetVT.SimpleTy != MVT::v8i16)
9338 return Register();
9339 if ((Subtarget->isNeonAvailable())) {
9340 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9341 }
9342 return Register();
9343}
9344
9345Register fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9346 if (RetVT.SimpleTy != MVT::v2i32)
9347 return Register();
9348 if ((Subtarget->isNeonAvailable())) {
9349 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9350 }
9351 return Register();
9352}
9353
9354Register fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9355 if (RetVT.SimpleTy != MVT::v4i32)
9356 return Register();
9357 if ((Subtarget->isNeonAvailable())) {
9358 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9359 }
9360 return Register();
9361}
9362
9363Register fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9364 if (RetVT.SimpleTy != MVT::v2i64)
9365 return Register();
9366 if ((Subtarget->isNeonAvailable())) {
9367 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9368 }
9369 return Register();
9370}
9371
9372Register fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9373 if (RetVT.SimpleTy != MVT::v4f16)
9374 return Register();
9375 if ((Subtarget->isNeonAvailable())) {
9376 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9377 }
9378 return Register();
9379}
9380
9381Register fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9382 if (RetVT.SimpleTy != MVT::v8f16)
9383 return Register();
9384 if ((Subtarget->isNeonAvailable())) {
9385 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9386 }
9387 return Register();
9388}
9389
9390Register fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9391 if (RetVT.SimpleTy != MVT::v4bf16)
9392 return Register();
9393 if ((Subtarget->isNeonAvailable())) {
9394 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9395 }
9396 return Register();
9397}
9398
9399Register fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9400 if (RetVT.SimpleTy != MVT::v8bf16)
9401 return Register();
9402 if ((Subtarget->isNeonAvailable())) {
9403 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9404 }
9405 return Register();
9406}
9407
9408Register fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9409 if (RetVT.SimpleTy != MVT::v2f32)
9410 return Register();
9411 if ((Subtarget->isNeonAvailable())) {
9412 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9413 }
9414 return Register();
9415}
9416
9417Register fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9418 if (RetVT.SimpleTy != MVT::v4f32)
9419 return Register();
9420 if ((Subtarget->isNeonAvailable())) {
9421 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9422 }
9423 return Register();
9424}
9425
9426Register fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9427 if (RetVT.SimpleTy != MVT::v2f64)
9428 return Register();
9429 if ((Subtarget->isNeonAvailable())) {
9430 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9431 }
9432 return Register();
9433}
9434
9435Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9436 if (RetVT.SimpleTy != MVT::nxv2i1)
9437 return Register();
9438 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9439 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9440 }
9441 return Register();
9442}
9443
9444Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9445 if (RetVT.SimpleTy != MVT::nxv4i1)
9446 return Register();
9447 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9448 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9449 }
9450 return Register();
9451}
9452
9453Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9454 if (RetVT.SimpleTy != MVT::nxv8i1)
9455 return Register();
9456 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9457 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9458 }
9459 return Register();
9460}
9461
9462Register fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9463 if (RetVT.SimpleTy != MVT::nxv16i1)
9464 return Register();
9465 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9466 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9467 }
9468 return Register();
9469}
9470
9471Register fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9472 if (RetVT.SimpleTy != MVT::nxv16i8)
9473 return Register();
9474 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9475 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9476 }
9477 return Register();
9478}
9479
9480Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9481 if (RetVT.SimpleTy != MVT::nxv8i16)
9482 return Register();
9483 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9484 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9485 }
9486 return Register();
9487}
9488
9489Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9490 if (RetVT.SimpleTy != MVT::nxv4i32)
9491 return Register();
9492 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9493 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9494 }
9495 return Register();
9496}
9497
9498Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9499 if (RetVT.SimpleTy != MVT::nxv2i64)
9500 return Register();
9501 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9502 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9503 }
9504 return Register();
9505}
9506
9507Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9508 if (RetVT.SimpleTy != MVT::nxv2f16)
9509 return Register();
9510 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9511 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9512 }
9513 return Register();
9514}
9515
9516Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9517 if (RetVT.SimpleTy != MVT::nxv4f16)
9518 return Register();
9519 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9520 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9521 }
9522 return Register();
9523}
9524
9525Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9526 if (RetVT.SimpleTy != MVT::nxv8f16)
9527 return Register();
9528 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9529 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9530 }
9531 return Register();
9532}
9533
9534Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9535 if (RetVT.SimpleTy != MVT::nxv2bf16)
9536 return Register();
9537 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9538 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9539 }
9540 return Register();
9541}
9542
9543Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9544 if (RetVT.SimpleTy != MVT::nxv4bf16)
9545 return Register();
9546 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9547 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9548 }
9549 return Register();
9550}
9551
9552Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9553 if (RetVT.SimpleTy != MVT::nxv8bf16)
9554 return Register();
9555 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9556 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9557 }
9558 return Register();
9559}
9560
9561Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9562 if (RetVT.SimpleTy != MVT::nxv2f32)
9563 return Register();
9564 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9565 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9566 }
9567 return Register();
9568}
9569
9570Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9571 if (RetVT.SimpleTy != MVT::nxv4f32)
9572 return Register();
9573 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9574 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9575 }
9576 return Register();
9577}
9578
9579Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9580 if (RetVT.SimpleTy != MVT::nxv2f64)
9581 return Register();
9582 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9583 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9584 }
9585 return Register();
9586}
9587
9588Register fastEmit_AArch64ISD_ZIP1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9589 switch (VT.SimpleTy) {
9590 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(RetVT, Op0, Op1);
9591 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(RetVT, Op0, Op1);
9592 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(RetVT, Op0, Op1);
9593 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(RetVT, Op0, Op1);
9594 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(RetVT, Op0, Op1);
9595 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(RetVT, Op0, Op1);
9596 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(RetVT, Op0, Op1);
9597 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(RetVT, Op0, Op1);
9598 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(RetVT, Op0, Op1);
9599 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(RetVT, Op0, Op1);
9600 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(RetVT, Op0, Op1);
9601 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(RetVT, Op0, Op1);
9602 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(RetVT, Op0, Op1);
9603 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(RetVT, Op0, Op1);
9604 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9605 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9606 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9607 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9608 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9609 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9610 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9611 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9612 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9613 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9614 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9615 case MVT::nxv2bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9616 case MVT::nxv4bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9617 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9618 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9619 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9620 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9621 default: return Register();
9622 }
9623}
9624
9625// FastEmit functions for AArch64ISD::ZIP2.
9626
9627Register fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9628 if (RetVT.SimpleTy != MVT::v8i8)
9629 return Register();
9630 if ((Subtarget->isNeonAvailable())) {
9631 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9632 }
9633 return Register();
9634}
9635
9636Register fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9637 if (RetVT.SimpleTy != MVT::v16i8)
9638 return Register();
9639 if ((Subtarget->isNeonAvailable())) {
9640 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9641 }
9642 return Register();
9643}
9644
9645Register fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9646 if (RetVT.SimpleTy != MVT::v4i16)
9647 return Register();
9648 if ((Subtarget->isNeonAvailable())) {
9649 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9650 }
9651 return Register();
9652}
9653
9654Register fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9655 if (RetVT.SimpleTy != MVT::v8i16)
9656 return Register();
9657 if ((Subtarget->isNeonAvailable())) {
9658 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9659 }
9660 return Register();
9661}
9662
9663Register fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9664 if (RetVT.SimpleTy != MVT::v2i32)
9665 return Register();
9666 if ((Subtarget->isNeonAvailable())) {
9667 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9668 }
9669 return Register();
9670}
9671
9672Register fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9673 if (RetVT.SimpleTy != MVT::v4i32)
9674 return Register();
9675 if ((Subtarget->isNeonAvailable())) {
9676 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9677 }
9678 return Register();
9679}
9680
9681Register fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9682 if (RetVT.SimpleTy != MVT::v2i64)
9683 return Register();
9684 if ((Subtarget->isNeonAvailable())) {
9685 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9686 }
9687 return Register();
9688}
9689
9690Register fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9691 if (RetVT.SimpleTy != MVT::v4f16)
9692 return Register();
9693 if ((Subtarget->isNeonAvailable())) {
9694 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9695 }
9696 return Register();
9697}
9698
9699Register fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9700 if (RetVT.SimpleTy != MVT::v8f16)
9701 return Register();
9702 if ((Subtarget->isNeonAvailable())) {
9703 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9704 }
9705 return Register();
9706}
9707
9708Register fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9709 if (RetVT.SimpleTy != MVT::v4bf16)
9710 return Register();
9711 if ((Subtarget->isNeonAvailable())) {
9712 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9713 }
9714 return Register();
9715}
9716
9717Register fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9718 if (RetVT.SimpleTy != MVT::v8bf16)
9719 return Register();
9720 if ((Subtarget->isNeonAvailable())) {
9721 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9722 }
9723 return Register();
9724}
9725
9726Register fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9727 if (RetVT.SimpleTy != MVT::v2f32)
9728 return Register();
9729 if ((Subtarget->isNeonAvailable())) {
9730 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9731 }
9732 return Register();
9733}
9734
9735Register fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9736 if (RetVT.SimpleTy != MVT::v4f32)
9737 return Register();
9738 if ((Subtarget->isNeonAvailable())) {
9739 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9740 }
9741 return Register();
9742}
9743
9744Register fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9745 if (RetVT.SimpleTy != MVT::v2f64)
9746 return Register();
9747 if ((Subtarget->isNeonAvailable())) {
9748 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9749 }
9750 return Register();
9751}
9752
9753Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9754 if (RetVT.SimpleTy != MVT::nxv2i1)
9755 return Register();
9756 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9757 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9758 }
9759 return Register();
9760}
9761
9762Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9763 if (RetVT.SimpleTy != MVT::nxv4i1)
9764 return Register();
9765 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9766 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9767 }
9768 return Register();
9769}
9770
9771Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9772 if (RetVT.SimpleTy != MVT::nxv8i1)
9773 return Register();
9774 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9775 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9776 }
9777 return Register();
9778}
9779
9780Register fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9781 if (RetVT.SimpleTy != MVT::nxv16i1)
9782 return Register();
9783 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9784 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9785 }
9786 return Register();
9787}
9788
9789Register fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9790 if (RetVT.SimpleTy != MVT::nxv16i8)
9791 return Register();
9792 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9793 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9794 }
9795 return Register();
9796}
9797
9798Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9799 if (RetVT.SimpleTy != MVT::nxv8i16)
9800 return Register();
9801 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9802 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9803 }
9804 return Register();
9805}
9806
9807Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9808 if (RetVT.SimpleTy != MVT::nxv4i32)
9809 return Register();
9810 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9811 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9812 }
9813 return Register();
9814}
9815
9816Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9817 if (RetVT.SimpleTy != MVT::nxv2i64)
9818 return Register();
9819 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9820 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9821 }
9822 return Register();
9823}
9824
9825Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9826 if (RetVT.SimpleTy != MVT::nxv2f16)
9827 return Register();
9828 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9829 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9830 }
9831 return Register();
9832}
9833
9834Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9835 if (RetVT.SimpleTy != MVT::nxv4f16)
9836 return Register();
9837 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9838 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9839 }
9840 return Register();
9841}
9842
9843Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9844 if (RetVT.SimpleTy != MVT::nxv8f16)
9845 return Register();
9846 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9847 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9848 }
9849 return Register();
9850}
9851
9852Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9853 if (RetVT.SimpleTy != MVT::nxv2bf16)
9854 return Register();
9855 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9856 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9857 }
9858 return Register();
9859}
9860
9861Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9862 if (RetVT.SimpleTy != MVT::nxv4bf16)
9863 return Register();
9864 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9865 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9866 }
9867 return Register();
9868}
9869
9870Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9871 if (RetVT.SimpleTy != MVT::nxv8bf16)
9872 return Register();
9873 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9874 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9875 }
9876 return Register();
9877}
9878
9879Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9880 if (RetVT.SimpleTy != MVT::nxv2f32)
9881 return Register();
9882 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9883 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9884 }
9885 return Register();
9886}
9887
9888Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9889 if (RetVT.SimpleTy != MVT::nxv4f32)
9890 return Register();
9891 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9892 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9893 }
9894 return Register();
9895}
9896
9897Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9898 if (RetVT.SimpleTy != MVT::nxv2f64)
9899 return Register();
9900 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9901 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9902 }
9903 return Register();
9904}
9905
9906Register fastEmit_AArch64ISD_ZIP2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9907 switch (VT.SimpleTy) {
9908 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(RetVT, Op0, Op1);
9909 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(RetVT, Op0, Op1);
9910 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(RetVT, Op0, Op1);
9911 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(RetVT, Op0, Op1);
9912 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(RetVT, Op0, Op1);
9913 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(RetVT, Op0, Op1);
9914 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(RetVT, Op0, Op1);
9915 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(RetVT, Op0, Op1);
9916 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(RetVT, Op0, Op1);
9917 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(RetVT, Op0, Op1);
9918 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(RetVT, Op0, Op1);
9919 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(RetVT, Op0, Op1);
9920 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(RetVT, Op0, Op1);
9921 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(RetVT, Op0, Op1);
9922 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9923 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9924 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9925 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9926 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9927 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9928 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9929 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9930 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9931 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9932 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9933 case MVT::nxv2bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9934 case MVT::nxv4bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9935 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9936 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9937 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9938 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9939 default: return Register();
9940 }
9941}
9942
9943// FastEmit functions for ISD::ABDS.
9944
9945Register fastEmit_ISD_ABDS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9946 if (RetVT.SimpleTy != MVT::v8i8)
9947 return Register();
9948 if ((Subtarget->isNeonAvailable())) {
9949 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9950 }
9951 return Register();
9952}
9953
9954Register fastEmit_ISD_ABDS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9955 if (RetVT.SimpleTy != MVT::v16i8)
9956 return Register();
9957 if ((Subtarget->isNeonAvailable())) {
9958 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9959 }
9960 return Register();
9961}
9962
9963Register fastEmit_ISD_ABDS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9964 if (RetVT.SimpleTy != MVT::v4i16)
9965 return Register();
9966 if ((Subtarget->isNeonAvailable())) {
9967 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9968 }
9969 return Register();
9970}
9971
9972Register fastEmit_ISD_ABDS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9973 if (RetVT.SimpleTy != MVT::v8i16)
9974 return Register();
9975 if ((Subtarget->isNeonAvailable())) {
9976 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9977 }
9978 return Register();
9979}
9980
9981Register fastEmit_ISD_ABDS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9982 if (RetVT.SimpleTy != MVT::v2i32)
9983 return Register();
9984 if ((Subtarget->isNeonAvailable())) {
9985 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9986 }
9987 return Register();
9988}
9989
9990Register fastEmit_ISD_ABDS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9991 if (RetVT.SimpleTy != MVT::v4i32)
9992 return Register();
9993 if ((Subtarget->isNeonAvailable())) {
9994 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9995 }
9996 return Register();
9997}
9998
9999Register fastEmit_ISD_ABDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10000 switch (VT.SimpleTy) {
10001 case MVT::v8i8: return fastEmit_ISD_ABDS_MVT_v8i8_rr(RetVT, Op0, Op1);
10002 case MVT::v16i8: return fastEmit_ISD_ABDS_MVT_v16i8_rr(RetVT, Op0, Op1);
10003 case MVT::v4i16: return fastEmit_ISD_ABDS_MVT_v4i16_rr(RetVT, Op0, Op1);
10004 case MVT::v8i16: return fastEmit_ISD_ABDS_MVT_v8i16_rr(RetVT, Op0, Op1);
10005 case MVT::v2i32: return fastEmit_ISD_ABDS_MVT_v2i32_rr(RetVT, Op0, Op1);
10006 case MVT::v4i32: return fastEmit_ISD_ABDS_MVT_v4i32_rr(RetVT, Op0, Op1);
10007 default: return Register();
10008 }
10009}
10010
10011// FastEmit functions for ISD::ABDU.
10012
10013Register fastEmit_ISD_ABDU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10014 if (RetVT.SimpleTy != MVT::v8i8)
10015 return Register();
10016 if ((Subtarget->isNeonAvailable())) {
10017 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10018 }
10019 return Register();
10020}
10021
10022Register fastEmit_ISD_ABDU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10023 if (RetVT.SimpleTy != MVT::v16i8)
10024 return Register();
10025 if ((Subtarget->isNeonAvailable())) {
10026 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10027 }
10028 return Register();
10029}
10030
10031Register fastEmit_ISD_ABDU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10032 if (RetVT.SimpleTy != MVT::v4i16)
10033 return Register();
10034 if ((Subtarget->isNeonAvailable())) {
10035 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10036 }
10037 return Register();
10038}
10039
10040Register fastEmit_ISD_ABDU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10041 if (RetVT.SimpleTy != MVT::v8i16)
10042 return Register();
10043 if ((Subtarget->isNeonAvailable())) {
10044 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10045 }
10046 return Register();
10047}
10048
10049Register fastEmit_ISD_ABDU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10050 if (RetVT.SimpleTy != MVT::v2i32)
10051 return Register();
10052 if ((Subtarget->isNeonAvailable())) {
10053 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10054 }
10055 return Register();
10056}
10057
10058Register fastEmit_ISD_ABDU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10059 if (RetVT.SimpleTy != MVT::v4i32)
10060 return Register();
10061 if ((Subtarget->isNeonAvailable())) {
10062 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10063 }
10064 return Register();
10065}
10066
10067Register fastEmit_ISD_ABDU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10068 switch (VT.SimpleTy) {
10069 case MVT::v8i8: return fastEmit_ISD_ABDU_MVT_v8i8_rr(RetVT, Op0, Op1);
10070 case MVT::v16i8: return fastEmit_ISD_ABDU_MVT_v16i8_rr(RetVT, Op0, Op1);
10071 case MVT::v4i16: return fastEmit_ISD_ABDU_MVT_v4i16_rr(RetVT, Op0, Op1);
10072 case MVT::v8i16: return fastEmit_ISD_ABDU_MVT_v8i16_rr(RetVT, Op0, Op1);
10073 case MVT::v2i32: return fastEmit_ISD_ABDU_MVT_v2i32_rr(RetVT, Op0, Op1);
10074 case MVT::v4i32: return fastEmit_ISD_ABDU_MVT_v4i32_rr(RetVT, Op0, Op1);
10075 default: return Register();
10076 }
10077}
10078
10079// FastEmit functions for ISD::ADD.
10080
10081Register fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10082 if (RetVT.SimpleTy != MVT::i32)
10083 return Register();
10084 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
10085}
10086
10087Register fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10088 if (RetVT.SimpleTy != MVT::i64)
10089 return Register();
10090 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
10091}
10092
10093Register fastEmit_ISD_ADD_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10094 if (RetVT.SimpleTy != MVT::v8i8)
10095 return Register();
10096 if ((Subtarget->isNeonAvailable())) {
10097 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10098 }
10099 return Register();
10100}
10101
10102Register fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10103 if (RetVT.SimpleTy != MVT::v16i8)
10104 return Register();
10105 if ((Subtarget->isNeonAvailable())) {
10106 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10107 }
10108 return Register();
10109}
10110
10111Register fastEmit_ISD_ADD_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10112 if (RetVT.SimpleTy != MVT::v4i16)
10113 return Register();
10114 if ((Subtarget->isNeonAvailable())) {
10115 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10116 }
10117 return Register();
10118}
10119
10120Register fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10121 if (RetVT.SimpleTy != MVT::v8i16)
10122 return Register();
10123 if ((Subtarget->isNeonAvailable())) {
10124 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10125 }
10126 return Register();
10127}
10128
10129Register fastEmit_ISD_ADD_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10130 if (RetVT.SimpleTy != MVT::v2i32)
10131 return Register();
10132 if ((Subtarget->isNeonAvailable())) {
10133 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10134 }
10135 return Register();
10136}
10137
10138Register fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10139 if (RetVT.SimpleTy != MVT::v4i32)
10140 return Register();
10141 if ((Subtarget->isNeonAvailable())) {
10142 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10143 }
10144 return Register();
10145}
10146
10147Register fastEmit_ISD_ADD_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
10148 if (RetVT.SimpleTy != MVT::v1i64)
10149 return Register();
10150 if ((Subtarget->isNeonAvailable())) {
10151 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
10152 }
10153 return Register();
10154}
10155
10156Register fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10157 if (RetVT.SimpleTy != MVT::v2i64)
10158 return Register();
10159 if ((Subtarget->isNeonAvailable())) {
10160 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10161 }
10162 return Register();
10163}
10164
10165Register fastEmit_ISD_ADD_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10166 if (RetVT.SimpleTy != MVT::nxv16i8)
10167 return Register();
10168 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10169 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
10170 }
10171 return Register();
10172}
10173
10174Register fastEmit_ISD_ADD_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10175 if (RetVT.SimpleTy != MVT::nxv8i16)
10176 return Register();
10177 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10178 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10179 }
10180 return Register();
10181}
10182
10183Register fastEmit_ISD_ADD_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10184 if (RetVT.SimpleTy != MVT::nxv4i32)
10185 return Register();
10186 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10187 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10188 }
10189 return Register();
10190}
10191
10192Register fastEmit_ISD_ADD_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10193 if (RetVT.SimpleTy != MVT::nxv2i64)
10194 return Register();
10195 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10196 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10197 }
10198 return Register();
10199}
10200
10201Register fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10202 switch (VT.SimpleTy) {
10203 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
10204 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op1);
10205 case MVT::v8i8: return fastEmit_ISD_ADD_MVT_v8i8_rr(RetVT, Op0, Op1);
10206 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
10207 case MVT::v4i16: return fastEmit_ISD_ADD_MVT_v4i16_rr(RetVT, Op0, Op1);
10208 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
10209 case MVT::v2i32: return fastEmit_ISD_ADD_MVT_v2i32_rr(RetVT, Op0, Op1);
10210 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
10211 case MVT::v1i64: return fastEmit_ISD_ADD_MVT_v1i64_rr(RetVT, Op0, Op1);
10212 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
10213 case MVT::nxv16i8: return fastEmit_ISD_ADD_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10214 case MVT::nxv8i16: return fastEmit_ISD_ADD_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10215 case MVT::nxv4i32: return fastEmit_ISD_ADD_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10216 case MVT::nxv2i64: return fastEmit_ISD_ADD_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10217 default: return Register();
10218 }
10219}
10220
10221// FastEmit functions for ISD::AND.
10222
10223Register fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10224 if (RetVT.SimpleTy != MVT::i32)
10225 return Register();
10226 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
10227}
10228
10229Register fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10230 if (RetVT.SimpleTy != MVT::i64)
10231 return Register();
10232 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
10233}
10234
10235Register fastEmit_ISD_AND_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10236 if (RetVT.SimpleTy != MVT::v8i8)
10237 return Register();
10238 if ((Subtarget->isNeonAvailable())) {
10239 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10240 }
10241 return Register();
10242}
10243
10244Register fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10245 if (RetVT.SimpleTy != MVT::v16i8)
10246 return Register();
10247 if ((Subtarget->isNeonAvailable())) {
10248 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10249 }
10250 return Register();
10251}
10252
10253Register fastEmit_ISD_AND_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10254 if (RetVT.SimpleTy != MVT::v4i16)
10255 return Register();
10256 if ((Subtarget->isNeonAvailable())) {
10257 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10258 }
10259 return Register();
10260}
10261
10262Register fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10263 if (RetVT.SimpleTy != MVT::v8i16)
10264 return Register();
10265 if ((Subtarget->isNeonAvailable())) {
10266 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10267 }
10268 return Register();
10269}
10270
10271Register fastEmit_ISD_AND_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10272 if (RetVT.SimpleTy != MVT::v2i32)
10273 return Register();
10274 if ((Subtarget->isNeonAvailable())) {
10275 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10276 }
10277 return Register();
10278}
10279
10280Register fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10281 if (RetVT.SimpleTy != MVT::v4i32)
10282 return Register();
10283 if ((Subtarget->isNeonAvailable())) {
10284 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10285 }
10286 return Register();
10287}
10288
10289Register fastEmit_ISD_AND_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
10290 if (RetVT.SimpleTy != MVT::v1i64)
10291 return Register();
10292 if ((Subtarget->isNeonAvailable())) {
10293 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10294 }
10295 return Register();
10296}
10297
10298Register fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10299 if (RetVT.SimpleTy != MVT::v2i64)
10300 return Register();
10301 if ((Subtarget->isNeonAvailable())) {
10302 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10303 }
10304 return Register();
10305}
10306
10307Register fastEmit_ISD_AND_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10308 if (RetVT.SimpleTy != MVT::nxv16i8)
10309 return Register();
10310 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10311 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10312 }
10313 return Register();
10314}
10315
10316Register fastEmit_ISD_AND_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10317 if (RetVT.SimpleTy != MVT::nxv8i16)
10318 return Register();
10319 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10320 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10321 }
10322 return Register();
10323}
10324
10325Register fastEmit_ISD_AND_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10326 if (RetVT.SimpleTy != MVT::nxv4i32)
10327 return Register();
10328 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10329 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10330 }
10331 return Register();
10332}
10333
10334Register fastEmit_ISD_AND_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10335 if (RetVT.SimpleTy != MVT::nxv2i64)
10336 return Register();
10337 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10338 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10339 }
10340 return Register();
10341}
10342
10343Register fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10344 switch (VT.SimpleTy) {
10345 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
10346 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op1);
10347 case MVT::v8i8: return fastEmit_ISD_AND_MVT_v8i8_rr(RetVT, Op0, Op1);
10348 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
10349 case MVT::v4i16: return fastEmit_ISD_AND_MVT_v4i16_rr(RetVT, Op0, Op1);
10350 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
10351 case MVT::v2i32: return fastEmit_ISD_AND_MVT_v2i32_rr(RetVT, Op0, Op1);
10352 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
10353 case MVT::v1i64: return fastEmit_ISD_AND_MVT_v1i64_rr(RetVT, Op0, Op1);
10354 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
10355 case MVT::nxv16i8: return fastEmit_ISD_AND_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10356 case MVT::nxv8i16: return fastEmit_ISD_AND_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10357 case MVT::nxv4i32: return fastEmit_ISD_AND_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10358 case MVT::nxv2i64: return fastEmit_ISD_AND_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10359 default: return Register();
10360 }
10361}
10362
10363// FastEmit functions for ISD::AVGCEILS.
10364
10365Register fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10366 if (RetVT.SimpleTy != MVT::v8i8)
10367 return Register();
10368 if ((Subtarget->isNeonAvailable())) {
10369 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10370 }
10371 return Register();
10372}
10373
10374Register fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10375 if (RetVT.SimpleTy != MVT::v16i8)
10376 return Register();
10377 if ((Subtarget->isNeonAvailable())) {
10378 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10379 }
10380 return Register();
10381}
10382
10383Register fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10384 if (RetVT.SimpleTy != MVT::v4i16)
10385 return Register();
10386 if ((Subtarget->isNeonAvailable())) {
10387 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10388 }
10389 return Register();
10390}
10391
10392Register fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10393 if (RetVT.SimpleTy != MVT::v8i16)
10394 return Register();
10395 if ((Subtarget->isNeonAvailable())) {
10396 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10397 }
10398 return Register();
10399}
10400
10401Register fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10402 if (RetVT.SimpleTy != MVT::v2i32)
10403 return Register();
10404 if ((Subtarget->isNeonAvailable())) {
10405 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10406 }
10407 return Register();
10408}
10409
10410Register fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10411 if (RetVT.SimpleTy != MVT::v4i32)
10412 return Register();
10413 if ((Subtarget->isNeonAvailable())) {
10414 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10415 }
10416 return Register();
10417}
10418
10419Register fastEmit_ISD_AVGCEILS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10420 switch (VT.SimpleTy) {
10421 case MVT::v8i8: return fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(RetVT, Op0, Op1);
10422 case MVT::v16i8: return fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(RetVT, Op0, Op1);
10423 case MVT::v4i16: return fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(RetVT, Op0, Op1);
10424 case MVT::v8i16: return fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(RetVT, Op0, Op1);
10425 case MVT::v2i32: return fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(RetVT, Op0, Op1);
10426 case MVT::v4i32: return fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(RetVT, Op0, Op1);
10427 default: return Register();
10428 }
10429}
10430
10431// FastEmit functions for ISD::AVGCEILU.
10432
10433Register fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10434 if (RetVT.SimpleTy != MVT::v8i8)
10435 return Register();
10436 if ((Subtarget->isNeonAvailable())) {
10437 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10438 }
10439 return Register();
10440}
10441
10442Register fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10443 if (RetVT.SimpleTy != MVT::v16i8)
10444 return Register();
10445 if ((Subtarget->isNeonAvailable())) {
10446 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10447 }
10448 return Register();
10449}
10450
10451Register fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10452 if (RetVT.SimpleTy != MVT::v4i16)
10453 return Register();
10454 if ((Subtarget->isNeonAvailable())) {
10455 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10456 }
10457 return Register();
10458}
10459
10460Register fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10461 if (RetVT.SimpleTy != MVT::v8i16)
10462 return Register();
10463 if ((Subtarget->isNeonAvailable())) {
10464 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10465 }
10466 return Register();
10467}
10468
10469Register fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10470 if (RetVT.SimpleTy != MVT::v2i32)
10471 return Register();
10472 if ((Subtarget->isNeonAvailable())) {
10473 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10474 }
10475 return Register();
10476}
10477
10478Register fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10479 if (RetVT.SimpleTy != MVT::v4i32)
10480 return Register();
10481 if ((Subtarget->isNeonAvailable())) {
10482 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10483 }
10484 return Register();
10485}
10486
10487Register fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10488 switch (VT.SimpleTy) {
10489 case MVT::v8i8: return fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(RetVT, Op0, Op1);
10490 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
10491 case MVT::v4i16: return fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(RetVT, Op0, Op1);
10492 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
10493 case MVT::v2i32: return fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(RetVT, Op0, Op1);
10494 case MVT::v4i32: return fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(RetVT, Op0, Op1);
10495 default: return Register();
10496 }
10497}
10498
10499// FastEmit functions for ISD::AVGFLOORS.
10500
10501Register fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10502 if (RetVT.SimpleTy != MVT::v8i8)
10503 return Register();
10504 if ((Subtarget->isNeonAvailable())) {
10505 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10506 }
10507 return Register();
10508}
10509
10510Register fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10511 if (RetVT.SimpleTy != MVT::v16i8)
10512 return Register();
10513 if ((Subtarget->isNeonAvailable())) {
10514 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10515 }
10516 return Register();
10517}
10518
10519Register fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10520 if (RetVT.SimpleTy != MVT::v4i16)
10521 return Register();
10522 if ((Subtarget->isNeonAvailable())) {
10523 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10524 }
10525 return Register();
10526}
10527
10528Register fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10529 if (RetVT.SimpleTy != MVT::v8i16)
10530 return Register();
10531 if ((Subtarget->isNeonAvailable())) {
10532 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10533 }
10534 return Register();
10535}
10536
10537Register fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10538 if (RetVT.SimpleTy != MVT::v2i32)
10539 return Register();
10540 if ((Subtarget->isNeonAvailable())) {
10541 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10542 }
10543 return Register();
10544}
10545
10546Register fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10547 if (RetVT.SimpleTy != MVT::v4i32)
10548 return Register();
10549 if ((Subtarget->isNeonAvailable())) {
10550 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10551 }
10552 return Register();
10553}
10554
10555Register fastEmit_ISD_AVGFLOORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10556 switch (VT.SimpleTy) {
10557 case MVT::v8i8: return fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(RetVT, Op0, Op1);
10558 case MVT::v16i8: return fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(RetVT, Op0, Op1);
10559 case MVT::v4i16: return fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(RetVT, Op0, Op1);
10560 case MVT::v8i16: return fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(RetVT, Op0, Op1);
10561 case MVT::v2i32: return fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(RetVT, Op0, Op1);
10562 case MVT::v4i32: return fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(RetVT, Op0, Op1);
10563 default: return Register();
10564 }
10565}
10566
10567// FastEmit functions for ISD::AVGFLOORU.
10568
10569Register fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10570 if (RetVT.SimpleTy != MVT::v8i8)
10571 return Register();
10572 if ((Subtarget->isNeonAvailable())) {
10573 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10574 }
10575 return Register();
10576}
10577
10578Register fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10579 if (RetVT.SimpleTy != MVT::v16i8)
10580 return Register();
10581 if ((Subtarget->isNeonAvailable())) {
10582 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10583 }
10584 return Register();
10585}
10586
10587Register fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10588 if (RetVT.SimpleTy != MVT::v4i16)
10589 return Register();
10590 if ((Subtarget->isNeonAvailable())) {
10591 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10592 }
10593 return Register();
10594}
10595
10596Register fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10597 if (RetVT.SimpleTy != MVT::v8i16)
10598 return Register();
10599 if ((Subtarget->isNeonAvailable())) {
10600 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10601 }
10602 return Register();
10603}
10604
10605Register fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10606 if (RetVT.SimpleTy != MVT::v2i32)
10607 return Register();
10608 if ((Subtarget->isNeonAvailable())) {
10609 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10610 }
10611 return Register();
10612}
10613
10614Register fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10615 if (RetVT.SimpleTy != MVT::v4i32)
10616 return Register();
10617 if ((Subtarget->isNeonAvailable())) {
10618 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10619 }
10620 return Register();
10621}
10622
10623Register fastEmit_ISD_AVGFLOORU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10624 switch (VT.SimpleTy) {
10625 case MVT::v8i8: return fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(RetVT, Op0, Op1);
10626 case MVT::v16i8: return fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(RetVT, Op0, Op1);
10627 case MVT::v4i16: return fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(RetVT, Op0, Op1);
10628 case MVT::v8i16: return fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(RetVT, Op0, Op1);
10629 case MVT::v2i32: return fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(RetVT, Op0, Op1);
10630 case MVT::v4i32: return fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(RetVT, Op0, Op1);
10631 default: return Register();
10632 }
10633}
10634
10635// FastEmit functions for ISD::CLMUL.
10636
10637Register fastEmit_ISD_CLMUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10638 if (RetVT.SimpleTy != MVT::v8i8)
10639 return Register();
10640 if ((Subtarget->isNeonAvailable())) {
10641 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10642 }
10643 return Register();
10644}
10645
10646Register fastEmit_ISD_CLMUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10647 if (RetVT.SimpleTy != MVT::v16i8)
10648 return Register();
10649 if ((Subtarget->isNeonAvailable())) {
10650 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10651 }
10652 return Register();
10653}
10654
10655Register fastEmit_ISD_CLMUL_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10656 if (RetVT.SimpleTy != MVT::nxv16i8)
10657 return Register();
10658 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2() || Subtarget->hasSME()))) {
10659 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMUL_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
10660 }
10661 return Register();
10662}
10663
10664Register fastEmit_ISD_CLMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10665 switch (VT.SimpleTy) {
10666 case MVT::v8i8: return fastEmit_ISD_CLMUL_MVT_v8i8_rr(RetVT, Op0, Op1);
10667 case MVT::v16i8: return fastEmit_ISD_CLMUL_MVT_v16i8_rr(RetVT, Op0, Op1);
10668 case MVT::nxv16i8: return fastEmit_ISD_CLMUL_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10669 default: return Register();
10670 }
10671}
10672
10673// FastEmit functions for ISD::CONCAT_VECTORS.
10674
10675Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(MVT RetVT, Register Op0, Register Op1) {
10676 if (RetVT.SimpleTy != MVT::nxv2i1)
10677 return Register();
10678 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10679 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
10680 }
10681 return Register();
10682}
10683
10684Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
10685 if (RetVT.SimpleTy != MVT::nxv4i1)
10686 return Register();
10687 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10688 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
10689 }
10690 return Register();
10691}
10692
10693Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
10694 if (RetVT.SimpleTy != MVT::nxv8i1)
10695 return Register();
10696 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10697 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
10698 }
10699 return Register();
10700}
10701
10702Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
10703 if (RetVT.SimpleTy != MVT::nxv16i1)
10704 return Register();
10705 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10706 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
10707 }
10708 return Register();
10709}
10710
10711Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
10712 if (RetVT.SimpleTy != MVT::nxv4f16)
10713 return Register();
10714 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10715 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10716 }
10717 return Register();
10718}
10719
10720Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10721 if (RetVT.SimpleTy != MVT::nxv8f16)
10722 return Register();
10723 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10724 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10725 }
10726 return Register();
10727}
10728
10729Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10730 if (RetVT.SimpleTy != MVT::nxv4bf16)
10731 return Register();
10732 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10733 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10734 }
10735 return Register();
10736}
10737
10738Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10739 if (RetVT.SimpleTy != MVT::nxv8bf16)
10740 return Register();
10741 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10742 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10743 }
10744 return Register();
10745}
10746
10747Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10748 if (RetVT.SimpleTy != MVT::nxv4f32)
10749 return Register();
10750 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10751 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10752 }
10753 return Register();
10754}
10755
10756Register fastEmit_ISD_CONCAT_VECTORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10757 switch (VT.SimpleTy) {
10758 case MVT::nxv1i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(RetVT, Op0, Op1);
10759 case MVT::nxv2i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(RetVT, Op0, Op1);
10760 case MVT::nxv4i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(RetVT, Op0, Op1);
10761 case MVT::nxv8i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(RetVT, Op0, Op1);
10762 case MVT::nxv2f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(RetVT, Op0, Op1);
10763 case MVT::nxv4f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(RetVT, Op0, Op1);
10764 case MVT::nxv2bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
10765 case MVT::nxv4bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
10766 case MVT::nxv2f32: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(RetVT, Op0, Op1);
10767 default: return Register();
10768 }
10769}
10770
10771// FastEmit functions for ISD::FADD.
10772
10773Register fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10774 if (RetVT.SimpleTy != MVT::f16)
10775 return Register();
10776 if ((Subtarget->hasFullFP16())) {
10777 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
10778 }
10779 return Register();
10780}
10781
10782Register fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10783 if (RetVT.SimpleTy != MVT::f32)
10784 return Register();
10785 if ((Subtarget->hasFPARMv8())) {
10786 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
10787 }
10788 return Register();
10789}
10790
10791Register fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
10792 if (RetVT.SimpleTy != MVT::f64)
10793 return Register();
10794 if ((Subtarget->hasFPARMv8())) {
10795 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10796 }
10797 return Register();
10798}
10799
10800Register fastEmit_ISD_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10801 if (RetVT.SimpleTy != MVT::v4f16)
10802 return Register();
10803 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10804 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10805 }
10806 return Register();
10807}
10808
10809Register fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10810 if (RetVT.SimpleTy != MVT::v8f16)
10811 return Register();
10812 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10813 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10814 }
10815 return Register();
10816}
10817
10818Register fastEmit_ISD_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10819 if (RetVT.SimpleTy != MVT::v2f32)
10820 return Register();
10821 if ((Subtarget->isNeonAvailable())) {
10822 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10823 }
10824 return Register();
10825}
10826
10827Register fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10828 if (RetVT.SimpleTy != MVT::v4f32)
10829 return Register();
10830 if ((Subtarget->isNeonAvailable())) {
10831 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10832 }
10833 return Register();
10834}
10835
10836Register fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10837 if (RetVT.SimpleTy != MVT::v2f64)
10838 return Register();
10839 if ((Subtarget->isNeonAvailable())) {
10840 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10841 }
10842 return Register();
10843}
10844
10845Register fastEmit_ISD_FADD_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10846 if (RetVT.SimpleTy != MVT::nxv8f16)
10847 return Register();
10848 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10849 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10850 }
10851 return Register();
10852}
10853
10854Register fastEmit_ISD_FADD_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10855 if (RetVT.SimpleTy != MVT::nxv8bf16)
10856 return Register();
10857 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
10858 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFADD_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10859 }
10860 return Register();
10861}
10862
10863Register fastEmit_ISD_FADD_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10864 if (RetVT.SimpleTy != MVT::nxv4f32)
10865 return Register();
10866 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10867 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10868 }
10869 return Register();
10870}
10871
10872Register fastEmit_ISD_FADD_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10873 if (RetVT.SimpleTy != MVT::nxv2f64)
10874 return Register();
10875 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10876 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10877 }
10878 return Register();
10879}
10880
10881Register fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10882 switch (VT.SimpleTy) {
10883 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
10884 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
10885 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
10886 case MVT::v4f16: return fastEmit_ISD_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
10887 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
10888 case MVT::v2f32: return fastEmit_ISD_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
10889 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
10890 case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
10891 case MVT::nxv8f16: return fastEmit_ISD_FADD_MVT_nxv8f16_rr(RetVT, Op0, Op1);
10892 case MVT::nxv8bf16: return fastEmit_ISD_FADD_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
10893 case MVT::nxv4f32: return fastEmit_ISD_FADD_MVT_nxv4f32_rr(RetVT, Op0, Op1);
10894 case MVT::nxv2f64: return fastEmit_ISD_FADD_MVT_nxv2f64_rr(RetVT, Op0, Op1);
10895 default: return Register();
10896 }
10897}
10898
10899// FastEmit functions for ISD::FDIV.
10900
10901Register fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10902 if (RetVT.SimpleTy != MVT::f16)
10903 return Register();
10904 if ((Subtarget->hasFullFP16())) {
10905 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
10906 }
10907 return Register();
10908}
10909
10910Register fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10911 if (RetVT.SimpleTy != MVT::f32)
10912 return Register();
10913 if ((Subtarget->hasFPARMv8())) {
10914 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
10915 }
10916 return Register();
10917}
10918
10919Register fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
10920 if (RetVT.SimpleTy != MVT::f64)
10921 return Register();
10922 if ((Subtarget->hasFPARMv8())) {
10923 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10924 }
10925 return Register();
10926}
10927
10928Register fastEmit_ISD_FDIV_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10929 if (RetVT.SimpleTy != MVT::v4f16)
10930 return Register();
10931 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10932 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10933 }
10934 return Register();
10935}
10936
10937Register fastEmit_ISD_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10938 if (RetVT.SimpleTy != MVT::v8f16)
10939 return Register();
10940 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10941 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10942 }
10943 return Register();
10944}
10945
10946Register fastEmit_ISD_FDIV_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10947 if (RetVT.SimpleTy != MVT::v2f32)
10948 return Register();
10949 if ((Subtarget->isNeonAvailable())) {
10950 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10951 }
10952 return Register();
10953}
10954
10955Register fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10956 if (RetVT.SimpleTy != MVT::v4f32)
10957 return Register();
10958 if ((Subtarget->isNeonAvailable())) {
10959 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10960 }
10961 return Register();
10962}
10963
10964Register fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10965 if (RetVT.SimpleTy != MVT::v2f64)
10966 return Register();
10967 if ((Subtarget->isNeonAvailable())) {
10968 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10969 }
10970 return Register();
10971}
10972
10973Register fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10974 switch (VT.SimpleTy) {
10975 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
10976 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
10977 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
10978 case MVT::v4f16: return fastEmit_ISD_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1);
10979 case MVT::v8f16: return fastEmit_ISD_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
10980 case MVT::v2f32: return fastEmit_ISD_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1);
10981 case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
10982 case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
10983 default: return Register();
10984 }
10985}
10986
10987// FastEmit functions for ISD::FMAXIMUM.
10988
10989Register fastEmit_ISD_FMAXIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10990 if (RetVT.SimpleTy != MVT::f16)
10991 return Register();
10992 if ((Subtarget->hasFullFP16())) {
10993 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
10994 }
10995 return Register();
10996}
10997
10998Register fastEmit_ISD_FMAXIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10999 if (RetVT.SimpleTy != MVT::f32)
11000 return Register();
11001 if ((Subtarget->hasFPARMv8())) {
11002 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11003 }
11004 return Register();
11005}
11006
11007Register fastEmit_ISD_FMAXIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11008 if (RetVT.SimpleTy != MVT::f64)
11009 return Register();
11010 if ((Subtarget->hasFPARMv8())) {
11011 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11012 }
11013 return Register();
11014}
11015
11016Register fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11017 if (RetVT.SimpleTy != MVT::v4f16)
11018 return Register();
11019 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11020 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11021 }
11022 return Register();
11023}
11024
11025Register fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11026 if (RetVT.SimpleTy != MVT::v8f16)
11027 return Register();
11028 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11029 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11030 }
11031 return Register();
11032}
11033
11034Register fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11035 if (RetVT.SimpleTy != MVT::v2f32)
11036 return Register();
11037 if ((Subtarget->isNeonAvailable())) {
11038 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11039 }
11040 return Register();
11041}
11042
11043Register fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11044 if (RetVT.SimpleTy != MVT::v4f32)
11045 return Register();
11046 if ((Subtarget->isNeonAvailable())) {
11047 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11048 }
11049 return Register();
11050}
11051
11052Register fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11053 if (RetVT.SimpleTy != MVT::v1f64)
11054 return Register();
11055 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11056}
11057
11058Register fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11059 if (RetVT.SimpleTy != MVT::v2f64)
11060 return Register();
11061 if ((Subtarget->isNeonAvailable())) {
11062 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11063 }
11064 return Register();
11065}
11066
11067Register fastEmit_ISD_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11068 switch (VT.SimpleTy) {
11069 case MVT::f16: return fastEmit_ISD_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1);
11070 case MVT::f32: return fastEmit_ISD_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1);
11071 case MVT::f64: return fastEmit_ISD_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1);
11072 case MVT::v4f16: return fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11073 case MVT::v8f16: return fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11074 case MVT::v2f32: return fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11075 case MVT::v4f32: return fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11076 case MVT::v1f64: return fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11077 case MVT::v2f64: return fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11078 default: return Register();
11079 }
11080}
11081
11082// FastEmit functions for ISD::FMAXNUM.
11083
11084Register fastEmit_ISD_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11085 if (RetVT.SimpleTy != MVT::f16)
11086 return Register();
11087 if ((Subtarget->hasFullFP16())) {
11088 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11089 }
11090 return Register();
11091}
11092
11093Register fastEmit_ISD_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11094 if (RetVT.SimpleTy != MVT::f32)
11095 return Register();
11096 if ((Subtarget->hasFPARMv8())) {
11097 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11098 }
11099 return Register();
11100}
11101
11102Register fastEmit_ISD_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11103 if (RetVT.SimpleTy != MVT::f64)
11104 return Register();
11105 if ((Subtarget->hasFPARMv8())) {
11106 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11107 }
11108 return Register();
11109}
11110
11111Register fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11112 if (RetVT.SimpleTy != MVT::v4f16)
11113 return Register();
11114 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11115 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11116 }
11117 return Register();
11118}
11119
11120Register fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11121 if (RetVT.SimpleTy != MVT::v8f16)
11122 return Register();
11123 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11124 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11125 }
11126 return Register();
11127}
11128
11129Register fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11130 if (RetVT.SimpleTy != MVT::v2f32)
11131 return Register();
11132 if ((Subtarget->isNeonAvailable())) {
11133 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11134 }
11135 return Register();
11136}
11137
11138Register fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11139 if (RetVT.SimpleTy != MVT::v4f32)
11140 return Register();
11141 if ((Subtarget->isNeonAvailable())) {
11142 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11143 }
11144 return Register();
11145}
11146
11147Register fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11148 if (RetVT.SimpleTy != MVT::v1f64)
11149 return Register();
11150 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11151}
11152
11153Register fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11154 if (RetVT.SimpleTy != MVT::v2f64)
11155 return Register();
11156 if ((Subtarget->isNeonAvailable())) {
11157 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11158 }
11159 return Register();
11160}
11161
11162Register fastEmit_ISD_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11163 switch (VT.SimpleTy) {
11164 case MVT::f16: return fastEmit_ISD_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
11165 case MVT::f32: return fastEmit_ISD_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
11166 case MVT::f64: return fastEmit_ISD_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
11167 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11168 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11169 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11170 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11171 case MVT::v1f64: return fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11172 case MVT::v2f64: return fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11173 default: return Register();
11174 }
11175}
11176
11177// FastEmit functions for ISD::FMAXNUM_IEEE.
11178
11179Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11180 if (RetVT.SimpleTy != MVT::f16)
11181 return Register();
11182 if ((Subtarget->hasFullFP16())) {
11183 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11184 }
11185 return Register();
11186}
11187
11188Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11189 if (RetVT.SimpleTy != MVT::f32)
11190 return Register();
11191 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11192}
11193
11194Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11195 if (RetVT.SimpleTy != MVT::f64)
11196 return Register();
11197 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11198}
11199
11200Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11201 if (RetVT.SimpleTy != MVT::v4f16)
11202 return Register();
11203 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11204 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11205 }
11206 return Register();
11207}
11208
11209Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11210 if (RetVT.SimpleTy != MVT::v8f16)
11211 return Register();
11212 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11213 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11214 }
11215 return Register();
11216}
11217
11218Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11219 if (RetVT.SimpleTy != MVT::v2f32)
11220 return Register();
11221 if ((Subtarget->isNeonAvailable())) {
11222 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11223 }
11224 return Register();
11225}
11226
11227Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11228 if (RetVT.SimpleTy != MVT::v4f32)
11229 return Register();
11230 if ((Subtarget->isNeonAvailable())) {
11231 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11232 }
11233 return Register();
11234}
11235
11236Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11237 if (RetVT.SimpleTy != MVT::v2f64)
11238 return Register();
11239 if ((Subtarget->isNeonAvailable())) {
11240 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11241 }
11242 return Register();
11243}
11244
11245Register fastEmit_ISD_FMAXNUM_IEEE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11246 switch (VT.SimpleTy) {
11247 case MVT::f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f16_rr(RetVT, Op0, Op1);
11248 case MVT::f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f32_rr(RetVT, Op0, Op1);
11249 case MVT::f64: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f64_rr(RetVT, Op0, Op1);
11250 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f16_rr(RetVT, Op0, Op1);
11251 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v8f16_rr(RetVT, Op0, Op1);
11252 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f32_rr(RetVT, Op0, Op1);
11253 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f32_rr(RetVT, Op0, Op1);
11254 case MVT::v2f64: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f64_rr(RetVT, Op0, Op1);
11255 default: return Register();
11256 }
11257}
11258
11259// FastEmit functions for ISD::FMINIMUM.
11260
11261Register fastEmit_ISD_FMINIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11262 if (RetVT.SimpleTy != MVT::f16)
11263 return Register();
11264 if ((Subtarget->hasFullFP16())) {
11265 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11266 }
11267 return Register();
11268}
11269
11270Register fastEmit_ISD_FMINIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11271 if (RetVT.SimpleTy != MVT::f32)
11272 return Register();
11273 if ((Subtarget->hasFPARMv8())) {
11274 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11275 }
11276 return Register();
11277}
11278
11279Register fastEmit_ISD_FMINIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11280 if (RetVT.SimpleTy != MVT::f64)
11281 return Register();
11282 if ((Subtarget->hasFPARMv8())) {
11283 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11284 }
11285 return Register();
11286}
11287
11288Register fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11289 if (RetVT.SimpleTy != MVT::v4f16)
11290 return Register();
11291 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11292 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11293 }
11294 return Register();
11295}
11296
11297Register fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11298 if (RetVT.SimpleTy != MVT::v8f16)
11299 return Register();
11300 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11301 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11302 }
11303 return Register();
11304}
11305
11306Register fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11307 if (RetVT.SimpleTy != MVT::v2f32)
11308 return Register();
11309 if ((Subtarget->isNeonAvailable())) {
11310 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11311 }
11312 return Register();
11313}
11314
11315Register fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11316 if (RetVT.SimpleTy != MVT::v4f32)
11317 return Register();
11318 if ((Subtarget->isNeonAvailable())) {
11319 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11320 }
11321 return Register();
11322}
11323
11324Register fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11325 if (RetVT.SimpleTy != MVT::v1f64)
11326 return Register();
11327 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11328}
11329
11330Register fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11331 if (RetVT.SimpleTy != MVT::v2f64)
11332 return Register();
11333 if ((Subtarget->isNeonAvailable())) {
11334 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11335 }
11336 return Register();
11337}
11338
11339Register fastEmit_ISD_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11340 switch (VT.SimpleTy) {
11341 case MVT::f16: return fastEmit_ISD_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1);
11342 case MVT::f32: return fastEmit_ISD_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1);
11343 case MVT::f64: return fastEmit_ISD_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1);
11344 case MVT::v4f16: return fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11345 case MVT::v8f16: return fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11346 case MVT::v2f32: return fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11347 case MVT::v4f32: return fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11348 case MVT::v1f64: return fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11349 case MVT::v2f64: return fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11350 default: return Register();
11351 }
11352}
11353
11354// FastEmit functions for ISD::FMINNUM.
11355
11356Register fastEmit_ISD_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11357 if (RetVT.SimpleTy != MVT::f16)
11358 return Register();
11359 if ((Subtarget->hasFullFP16())) {
11360 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11361 }
11362 return Register();
11363}
11364
11365Register fastEmit_ISD_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11366 if (RetVT.SimpleTy != MVT::f32)
11367 return Register();
11368 if ((Subtarget->hasFPARMv8())) {
11369 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11370 }
11371 return Register();
11372}
11373
11374Register fastEmit_ISD_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11375 if (RetVT.SimpleTy != MVT::f64)
11376 return Register();
11377 if ((Subtarget->hasFPARMv8())) {
11378 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11379 }
11380 return Register();
11381}
11382
11383Register fastEmit_ISD_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11384 if (RetVT.SimpleTy != MVT::v4f16)
11385 return Register();
11386 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11387 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11388 }
11389 return Register();
11390}
11391
11392Register fastEmit_ISD_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11393 if (RetVT.SimpleTy != MVT::v8f16)
11394 return Register();
11395 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11396 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11397 }
11398 return Register();
11399}
11400
11401Register fastEmit_ISD_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11402 if (RetVT.SimpleTy != MVT::v2f32)
11403 return Register();
11404 if ((Subtarget->isNeonAvailable())) {
11405 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11406 }
11407 return Register();
11408}
11409
11410Register fastEmit_ISD_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11411 if (RetVT.SimpleTy != MVT::v4f32)
11412 return Register();
11413 if ((Subtarget->isNeonAvailable())) {
11414 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11415 }
11416 return Register();
11417}
11418
11419Register fastEmit_ISD_FMINNUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11420 if (RetVT.SimpleTy != MVT::v1f64)
11421 return Register();
11422 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11423}
11424
11425Register fastEmit_ISD_FMINNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11426 if (RetVT.SimpleTy != MVT::v2f64)
11427 return Register();
11428 if ((Subtarget->isNeonAvailable())) {
11429 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11430 }
11431 return Register();
11432}
11433
11434Register fastEmit_ISD_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11435 switch (VT.SimpleTy) {
11436 case MVT::f16: return fastEmit_ISD_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
11437 case MVT::f32: return fastEmit_ISD_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
11438 case MVT::f64: return fastEmit_ISD_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
11439 case MVT::v4f16: return fastEmit_ISD_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11440 case MVT::v8f16: return fastEmit_ISD_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11441 case MVT::v2f32: return fastEmit_ISD_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11442 case MVT::v4f32: return fastEmit_ISD_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11443 case MVT::v1f64: return fastEmit_ISD_FMINNUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11444 case MVT::v2f64: return fastEmit_ISD_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11445 default: return Register();
11446 }
11447}
11448
11449// FastEmit functions for ISD::FMINNUM_IEEE.
11450
11451Register fastEmit_ISD_FMINNUM_IEEE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11452 if (RetVT.SimpleTy != MVT::f16)
11453 return Register();
11454 if ((Subtarget->hasFullFP16())) {
11455 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11456 }
11457 return Register();
11458}
11459
11460Register fastEmit_ISD_FMINNUM_IEEE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11461 if (RetVT.SimpleTy != MVT::f32)
11462 return Register();
11463 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11464}
11465
11466Register fastEmit_ISD_FMINNUM_IEEE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11467 if (RetVT.SimpleTy != MVT::f64)
11468 return Register();
11469 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11470}
11471
11472Register fastEmit_ISD_FMINNUM_IEEE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11473 if (RetVT.SimpleTy != MVT::v4f16)
11474 return Register();
11475 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11476 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11477 }
11478 return Register();
11479}
11480
11481Register fastEmit_ISD_FMINNUM_IEEE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11482 if (RetVT.SimpleTy != MVT::v8f16)
11483 return Register();
11484 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11485 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11486 }
11487 return Register();
11488}
11489
11490Register fastEmit_ISD_FMINNUM_IEEE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11491 if (RetVT.SimpleTy != MVT::v2f32)
11492 return Register();
11493 if ((Subtarget->isNeonAvailable())) {
11494 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11495 }
11496 return Register();
11497}
11498
11499Register fastEmit_ISD_FMINNUM_IEEE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11500 if (RetVT.SimpleTy != MVT::v4f32)
11501 return Register();
11502 if ((Subtarget->isNeonAvailable())) {
11503 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11504 }
11505 return Register();
11506}
11507
11508Register fastEmit_ISD_FMINNUM_IEEE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11509 if (RetVT.SimpleTy != MVT::v2f64)
11510 return Register();
11511 if ((Subtarget->isNeonAvailable())) {
11512 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11513 }
11514 return Register();
11515}
11516
11517Register fastEmit_ISD_FMINNUM_IEEE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11518 switch (VT.SimpleTy) {
11519 case MVT::f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_f16_rr(RetVT, Op0, Op1);
11520 case MVT::f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_f32_rr(RetVT, Op0, Op1);
11521 case MVT::f64: return fastEmit_ISD_FMINNUM_IEEE_MVT_f64_rr(RetVT, Op0, Op1);
11522 case MVT::v4f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_v4f16_rr(RetVT, Op0, Op1);
11523 case MVT::v8f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_v8f16_rr(RetVT, Op0, Op1);
11524 case MVT::v2f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_v2f32_rr(RetVT, Op0, Op1);
11525 case MVT::v4f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_v4f32_rr(RetVT, Op0, Op1);
11526 case MVT::v2f64: return fastEmit_ISD_FMINNUM_IEEE_MVT_v2f64_rr(RetVT, Op0, Op1);
11527 default: return Register();
11528 }
11529}
11530
11531// FastEmit functions for ISD::FMUL.
11532
11533Register fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11534 if (RetVT.SimpleTy != MVT::f16)
11535 return Register();
11536 if ((Subtarget->hasFullFP16())) {
11537 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11538 }
11539 return Register();
11540}
11541
11542Register fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11543 if (RetVT.SimpleTy != MVT::f32)
11544 return Register();
11545 if ((Subtarget->hasFPARMv8())) {
11546 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11547 }
11548 return Register();
11549}
11550
11551Register fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11552 if (RetVT.SimpleTy != MVT::f64)
11553 return Register();
11554 if ((Subtarget->hasFPARMv8())) {
11555 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11556 }
11557 return Register();
11558}
11559
11560Register fastEmit_ISD_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11561 if (RetVT.SimpleTy != MVT::v4f16)
11562 return Register();
11563 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11564 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11565 }
11566 return Register();
11567}
11568
11569Register fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11570 if (RetVT.SimpleTy != MVT::v8f16)
11571 return Register();
11572 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11573 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11574 }
11575 return Register();
11576}
11577
11578Register fastEmit_ISD_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11579 if (RetVT.SimpleTy != MVT::v2f32)
11580 return Register();
11581 if ((Subtarget->isNeonAvailable())) {
11582 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11583 }
11584 return Register();
11585}
11586
11587Register fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11588 if (RetVT.SimpleTy != MVT::v4f32)
11589 return Register();
11590 if ((Subtarget->isNeonAvailable())) {
11591 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11592 }
11593 return Register();
11594}
11595
11596Register fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11597 if (RetVT.SimpleTy != MVT::v2f64)
11598 return Register();
11599 if ((Subtarget->isNeonAvailable())) {
11600 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11601 }
11602 return Register();
11603}
11604
11605Register fastEmit_ISD_FMUL_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11606 if (RetVT.SimpleTy != MVT::nxv8f16)
11607 return Register();
11608 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11609 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11610 }
11611 return Register();
11612}
11613
11614Register fastEmit_ISD_FMUL_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11615 if (RetVT.SimpleTy != MVT::nxv8bf16)
11616 return Register();
11617 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
11618 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFMUL_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11619 }
11620 return Register();
11621}
11622
11623Register fastEmit_ISD_FMUL_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11624 if (RetVT.SimpleTy != MVT::nxv4f32)
11625 return Register();
11626 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11627 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11628 }
11629 return Register();
11630}
11631
11632Register fastEmit_ISD_FMUL_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11633 if (RetVT.SimpleTy != MVT::nxv2f64)
11634 return Register();
11635 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11636 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11637 }
11638 return Register();
11639}
11640
11641Register fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11642 switch (VT.SimpleTy) {
11643 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
11644 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
11645 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
11646 case MVT::v4f16: return fastEmit_ISD_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
11647 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
11648 case MVT::v2f32: return fastEmit_ISD_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
11649 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
11650 case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
11651 case MVT::nxv8f16: return fastEmit_ISD_FMUL_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11652 case MVT::nxv8bf16: return fastEmit_ISD_FMUL_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11653 case MVT::nxv4f32: return fastEmit_ISD_FMUL_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11654 case MVT::nxv2f64: return fastEmit_ISD_FMUL_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11655 default: return Register();
11656 }
11657}
11658
11659// FastEmit functions for ISD::FSUB.
11660
11661Register fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11662 if (RetVT.SimpleTy != MVT::f16)
11663 return Register();
11664 if ((Subtarget->hasFullFP16())) {
11665 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11666 }
11667 return Register();
11668}
11669
11670Register fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11671 if (RetVT.SimpleTy != MVT::f32)
11672 return Register();
11673 if ((Subtarget->hasFPARMv8())) {
11674 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11675 }
11676 return Register();
11677}
11678
11679Register fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11680 if (RetVT.SimpleTy != MVT::f64)
11681 return Register();
11682 if ((Subtarget->hasFPARMv8())) {
11683 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11684 }
11685 return Register();
11686}
11687
11688Register fastEmit_ISD_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11689 if (RetVT.SimpleTy != MVT::v4f16)
11690 return Register();
11691 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11692 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11693 }
11694 return Register();
11695}
11696
11697Register fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11698 if (RetVT.SimpleTy != MVT::v8f16)
11699 return Register();
11700 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11701 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11702 }
11703 return Register();
11704}
11705
11706Register fastEmit_ISD_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11707 if (RetVT.SimpleTy != MVT::v2f32)
11708 return Register();
11709 if ((Subtarget->isNeonAvailable())) {
11710 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11711 }
11712 return Register();
11713}
11714
11715Register fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11716 if (RetVT.SimpleTy != MVT::v4f32)
11717 return Register();
11718 if ((Subtarget->isNeonAvailable())) {
11719 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11720 }
11721 return Register();
11722}
11723
11724Register fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11725 if (RetVT.SimpleTy != MVT::v2f64)
11726 return Register();
11727 if ((Subtarget->isNeonAvailable())) {
11728 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11729 }
11730 return Register();
11731}
11732
11733Register fastEmit_ISD_FSUB_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11734 if (RetVT.SimpleTy != MVT::nxv8f16)
11735 return Register();
11736 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11737 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11738 }
11739 return Register();
11740}
11741
11742Register fastEmit_ISD_FSUB_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11743 if (RetVT.SimpleTy != MVT::nxv8bf16)
11744 return Register();
11745 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
11746 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFSUB_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11747 }
11748 return Register();
11749}
11750
11751Register fastEmit_ISD_FSUB_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11752 if (RetVT.SimpleTy != MVT::nxv4f32)
11753 return Register();
11754 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11755 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11756 }
11757 return Register();
11758}
11759
11760Register fastEmit_ISD_FSUB_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11761 if (RetVT.SimpleTy != MVT::nxv2f64)
11762 return Register();
11763 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11764 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11765 }
11766 return Register();
11767}
11768
11769Register fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11770 switch (VT.SimpleTy) {
11771 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
11772 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
11773 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
11774 case MVT::v4f16: return fastEmit_ISD_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
11775 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
11776 case MVT::v2f32: return fastEmit_ISD_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
11777 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
11778 case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
11779 case MVT::nxv8f16: return fastEmit_ISD_FSUB_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11780 case MVT::nxv8bf16: return fastEmit_ISD_FSUB_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11781 case MVT::nxv4f32: return fastEmit_ISD_FSUB_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11782 case MVT::nxv2f64: return fastEmit_ISD_FSUB_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11783 default: return Register();
11784 }
11785}
11786
11787// FastEmit functions for ISD::GET_ACTIVE_LANE_MASK.
11788
11789Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv2i1_rr(Register Op0, Register Op1) {
11790 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11791 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_D, RC: &AArch64::PPRRegClass, Op0, Op1);
11792 }
11793 return Register();
11794}
11795
11796Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv4i1_rr(Register Op0, Register Op1) {
11797 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11798 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_S, RC: &AArch64::PPRRegClass, Op0, Op1);
11799 }
11800 return Register();
11801}
11802
11803Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv8i1_rr(Register Op0, Register Op1) {
11804 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11805 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_H, RC: &AArch64::PPRRegClass, Op0, Op1);
11806 }
11807 return Register();
11808}
11809
11810Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv16i1_rr(Register Op0, Register Op1) {
11811 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11812 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_B, RC: &AArch64::PPRRegClass, Op0, Op1);
11813 }
11814 return Register();
11815}
11816
11817Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
11818switch (RetVT.SimpleTy) {
11819 case MVT::nxv2i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv2i1_rr(Op0, Op1);
11820 case MVT::nxv4i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv4i1_rr(Op0, Op1);
11821 case MVT::nxv8i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv8i1_rr(Op0, Op1);
11822 case MVT::nxv16i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv16i1_rr(Op0, Op1);
11823 default: return Register();
11824}
11825}
11826
11827Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv2i1_rr(Register Op0, Register Op1) {
11828 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11829 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_D, RC: &AArch64::PPRRegClass, Op0, Op1);
11830 }
11831 return Register();
11832}
11833
11834Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv4i1_rr(Register Op0, Register Op1) {
11835 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11836 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_S, RC: &AArch64::PPRRegClass, Op0, Op1);
11837 }
11838 return Register();
11839}
11840
11841Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv8i1_rr(Register Op0, Register Op1) {
11842 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11843 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_H, RC: &AArch64::PPRRegClass, Op0, Op1);
11844 }
11845 return Register();
11846}
11847
11848Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv16i1_rr(Register Op0, Register Op1) {
11849 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11850 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_B, RC: &AArch64::PPRRegClass, Op0, Op1);
11851 }
11852 return Register();
11853}
11854
11855Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11856switch (RetVT.SimpleTy) {
11857 case MVT::nxv2i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv2i1_rr(Op0, Op1);
11858 case MVT::nxv4i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv4i1_rr(Op0, Op1);
11859 case MVT::nxv8i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv8i1_rr(Op0, Op1);
11860 case MVT::nxv16i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv16i1_rr(Op0, Op1);
11861 default: return Register();
11862}
11863}
11864
11865Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11866 switch (VT.SimpleTy) {
11867 case MVT::i32: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_rr(RetVT, Op0, Op1);
11868 case MVT::i64: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_rr(RetVT, Op0, Op1);
11869 default: return Register();
11870 }
11871}
11872
11873// FastEmit functions for ISD::MUL.
11874
11875Register fastEmit_ISD_MUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
11876 if (RetVT.SimpleTy != MVT::v8i8)
11877 return Register();
11878 if ((Subtarget->isNeonAvailable())) {
11879 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
11880 }
11881 return Register();
11882}
11883
11884Register fastEmit_ISD_MUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
11885 if (RetVT.SimpleTy != MVT::v16i8)
11886 return Register();
11887 if ((Subtarget->isNeonAvailable())) {
11888 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
11889 }
11890 return Register();
11891}
11892
11893Register fastEmit_ISD_MUL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
11894 if (RetVT.SimpleTy != MVT::v4i16)
11895 return Register();
11896 if ((Subtarget->isNeonAvailable())) {
11897 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11898 }
11899 return Register();
11900}
11901
11902Register fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
11903 if (RetVT.SimpleTy != MVT::v8i16)
11904 return Register();
11905 if ((Subtarget->isNeonAvailable())) {
11906 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11907 }
11908 return Register();
11909}
11910
11911Register fastEmit_ISD_MUL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
11912 if (RetVT.SimpleTy != MVT::v2i32)
11913 return Register();
11914 if ((Subtarget->isNeonAvailable())) {
11915 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11916 }
11917 return Register();
11918}
11919
11920Register fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
11921 if (RetVT.SimpleTy != MVT::v4i32)
11922 return Register();
11923 if ((Subtarget->isNeonAvailable())) {
11924 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11925 }
11926 return Register();
11927}
11928
11929Register fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11930 switch (VT.SimpleTy) {
11931 case MVT::v8i8: return fastEmit_ISD_MUL_MVT_v8i8_rr(RetVT, Op0, Op1);
11932 case MVT::v16i8: return fastEmit_ISD_MUL_MVT_v16i8_rr(RetVT, Op0, Op1);
11933 case MVT::v4i16: return fastEmit_ISD_MUL_MVT_v4i16_rr(RetVT, Op0, Op1);
11934 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
11935 case MVT::v2i32: return fastEmit_ISD_MUL_MVT_v2i32_rr(RetVT, Op0, Op1);
11936 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
11937 default: return Register();
11938 }
11939}
11940
11941// FastEmit functions for ISD::MULHS.
11942
11943Register fastEmit_ISD_MULHS_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11944 if (RetVT.SimpleTy != MVT::i64)
11945 return Register();
11946 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULHrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
11947}
11948
11949Register fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11950 switch (VT.SimpleTy) {
11951 case MVT::i64: return fastEmit_ISD_MULHS_MVT_i64_rr(RetVT, Op0, Op1);
11952 default: return Register();
11953 }
11954}
11955
11956// FastEmit functions for ISD::MULHU.
11957
11958Register fastEmit_ISD_MULHU_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11959 if (RetVT.SimpleTy != MVT::i64)
11960 return Register();
11961 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULHrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
11962}
11963
11964Register fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11965 switch (VT.SimpleTy) {
11966 case MVT::i64: return fastEmit_ISD_MULHU_MVT_i64_rr(RetVT, Op0, Op1);
11967 default: return Register();
11968 }
11969}
11970
11971// FastEmit functions for ISD::OR.
11972
11973Register fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
11974 if (RetVT.SimpleTy != MVT::i32)
11975 return Register();
11976 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
11977}
11978
11979Register fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11980 if (RetVT.SimpleTy != MVT::i64)
11981 return Register();
11982 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
11983}
11984
11985Register fastEmit_ISD_OR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
11986 if (RetVT.SimpleTy != MVT::v8i8)
11987 return Register();
11988 if ((Subtarget->isNeonAvailable())) {
11989 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
11990 }
11991 return Register();
11992}
11993
11994Register fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
11995 if (RetVT.SimpleTy != MVT::v16i8)
11996 return Register();
11997 if ((Subtarget->isNeonAvailable())) {
11998 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
11999 }
12000 return Register();
12001}
12002
12003Register fastEmit_ISD_OR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12004 if (RetVT.SimpleTy != MVT::v4i16)
12005 return Register();
12006 if ((Subtarget->isNeonAvailable())) {
12007 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12008 }
12009 return Register();
12010}
12011
12012Register fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12013 if (RetVT.SimpleTy != MVT::v8i16)
12014 return Register();
12015 if ((Subtarget->isNeonAvailable())) {
12016 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12017 }
12018 return Register();
12019}
12020
12021Register fastEmit_ISD_OR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12022 if (RetVT.SimpleTy != MVT::v2i32)
12023 return Register();
12024 if ((Subtarget->isNeonAvailable())) {
12025 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12026 }
12027 return Register();
12028}
12029
12030Register fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12031 if (RetVT.SimpleTy != MVT::v4i32)
12032 return Register();
12033 if ((Subtarget->isNeonAvailable())) {
12034 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12035 }
12036 return Register();
12037}
12038
12039Register fastEmit_ISD_OR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12040 if (RetVT.SimpleTy != MVT::v1i64)
12041 return Register();
12042 if ((Subtarget->isNeonAvailable())) {
12043 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12044 }
12045 return Register();
12046}
12047
12048Register fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12049 if (RetVT.SimpleTy != MVT::v2i64)
12050 return Register();
12051 if ((Subtarget->isNeonAvailable())) {
12052 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12053 }
12054 return Register();
12055}
12056
12057Register fastEmit_ISD_OR_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12058 if (RetVT.SimpleTy != MVT::nxv16i8)
12059 return Register();
12060 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12061 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12062 }
12063 return Register();
12064}
12065
12066Register fastEmit_ISD_OR_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12067 if (RetVT.SimpleTy != MVT::nxv8i16)
12068 return Register();
12069 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12070 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12071 }
12072 return Register();
12073}
12074
12075Register fastEmit_ISD_OR_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12076 if (RetVT.SimpleTy != MVT::nxv4i32)
12077 return Register();
12078 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12079 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12080 }
12081 return Register();
12082}
12083
12084Register fastEmit_ISD_OR_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12085 if (RetVT.SimpleTy != MVT::nxv2i64)
12086 return Register();
12087 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12088 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12089 }
12090 return Register();
12091}
12092
12093Register fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12094 switch (VT.SimpleTy) {
12095 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
12096 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op1);
12097 case MVT::v8i8: return fastEmit_ISD_OR_MVT_v8i8_rr(RetVT, Op0, Op1);
12098 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
12099 case MVT::v4i16: return fastEmit_ISD_OR_MVT_v4i16_rr(RetVT, Op0, Op1);
12100 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
12101 case MVT::v2i32: return fastEmit_ISD_OR_MVT_v2i32_rr(RetVT, Op0, Op1);
12102 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
12103 case MVT::v1i64: return fastEmit_ISD_OR_MVT_v1i64_rr(RetVT, Op0, Op1);
12104 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
12105 case MVT::nxv16i8: return fastEmit_ISD_OR_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12106 case MVT::nxv8i16: return fastEmit_ISD_OR_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12107 case MVT::nxv4i32: return fastEmit_ISD_OR_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12108 case MVT::nxv2i64: return fastEmit_ISD_OR_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12109 default: return Register();
12110 }
12111}
12112
12113// FastEmit functions for ISD::ROTR.
12114
12115Register fastEmit_ISD_ROTR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12116 if (RetVT.SimpleTy != MVT::i64)
12117 return Register();
12118 return fastEmitInst_rr(MachineInstOpcode: AArch64::RORVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12119}
12120
12121Register fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12122 switch (VT.SimpleTy) {
12123 case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_rr(RetVT, Op0, Op1);
12124 default: return Register();
12125 }
12126}
12127
12128// FastEmit functions for ISD::SADDSAT.
12129
12130Register fastEmit_ISD_SADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12131 if (RetVT.SimpleTy != MVT::v8i8)
12132 return Register();
12133 if ((Subtarget->isNeonAvailable())) {
12134 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12135 }
12136 return Register();
12137}
12138
12139Register fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12140 if (RetVT.SimpleTy != MVT::v16i8)
12141 return Register();
12142 if ((Subtarget->isNeonAvailable())) {
12143 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12144 }
12145 return Register();
12146}
12147
12148Register fastEmit_ISD_SADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12149 if (RetVT.SimpleTy != MVT::v4i16)
12150 return Register();
12151 if ((Subtarget->isNeonAvailable())) {
12152 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12153 }
12154 return Register();
12155}
12156
12157Register fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12158 if (RetVT.SimpleTy != MVT::v8i16)
12159 return Register();
12160 if ((Subtarget->isNeonAvailable())) {
12161 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12162 }
12163 return Register();
12164}
12165
12166Register fastEmit_ISD_SADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12167 if (RetVT.SimpleTy != MVT::v2i32)
12168 return Register();
12169 if ((Subtarget->isNeonAvailable())) {
12170 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12171 }
12172 return Register();
12173}
12174
12175Register fastEmit_ISD_SADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12176 if (RetVT.SimpleTy != MVT::v4i32)
12177 return Register();
12178 if ((Subtarget->isNeonAvailable())) {
12179 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12180 }
12181 return Register();
12182}
12183
12184Register fastEmit_ISD_SADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12185 if (RetVT.SimpleTy != MVT::v1i64)
12186 return Register();
12187 if ((Subtarget->isNeonAvailable())) {
12188 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
12189 }
12190 return Register();
12191}
12192
12193Register fastEmit_ISD_SADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12194 if (RetVT.SimpleTy != MVT::v2i64)
12195 return Register();
12196 if ((Subtarget->isNeonAvailable())) {
12197 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12198 }
12199 return Register();
12200}
12201
12202Register fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12203 if (RetVT.SimpleTy != MVT::nxv16i8)
12204 return Register();
12205 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12206 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
12207 }
12208 return Register();
12209}
12210
12211Register fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12212 if (RetVT.SimpleTy != MVT::nxv8i16)
12213 return Register();
12214 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12215 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
12216 }
12217 return Register();
12218}
12219
12220Register fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12221 if (RetVT.SimpleTy != MVT::nxv4i32)
12222 return Register();
12223 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12224 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
12225 }
12226 return Register();
12227}
12228
12229Register fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12230 if (RetVT.SimpleTy != MVT::nxv2i64)
12231 return Register();
12232 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12233 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
12234 }
12235 return Register();
12236}
12237
12238Register fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12239 switch (VT.SimpleTy) {
12240 case MVT::v8i8: return fastEmit_ISD_SADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
12241 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12242 case MVT::v4i16: return fastEmit_ISD_SADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
12243 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12244 case MVT::v2i32: return fastEmit_ISD_SADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
12245 case MVT::v4i32: return fastEmit_ISD_SADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
12246 case MVT::v1i64: return fastEmit_ISD_SADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
12247 case MVT::v2i64: return fastEmit_ISD_SADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
12248 case MVT::nxv16i8: return fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12249 case MVT::nxv8i16: return fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12250 case MVT::nxv4i32: return fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12251 case MVT::nxv2i64: return fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12252 default: return Register();
12253 }
12254}
12255
12256// FastEmit functions for ISD::SDIV.
12257
12258Register fastEmit_ISD_SDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12259 if (RetVT.SimpleTy != MVT::i32)
12260 return Register();
12261 return fastEmitInst_rr(MachineInstOpcode: AArch64::SDIVWr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12262}
12263
12264Register fastEmit_ISD_SDIV_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12265 if (RetVT.SimpleTy != MVT::i64)
12266 return Register();
12267 return fastEmitInst_rr(MachineInstOpcode: AArch64::SDIVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12268}
12269
12270Register fastEmit_ISD_SDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12271 switch (VT.SimpleTy) {
12272 case MVT::i32: return fastEmit_ISD_SDIV_MVT_i32_rr(RetVT, Op0, Op1);
12273 case MVT::i64: return fastEmit_ISD_SDIV_MVT_i64_rr(RetVT, Op0, Op1);
12274 default: return Register();
12275 }
12276}
12277
12278// FastEmit functions for ISD::SHL.
12279
12280Register fastEmit_ISD_SHL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12281 if (RetVT.SimpleTy != MVT::i64)
12282 return Register();
12283 return fastEmitInst_rr(MachineInstOpcode: AArch64::LSLVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12284}
12285
12286Register fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12287 switch (VT.SimpleTy) {
12288 case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_rr(RetVT, Op0, Op1);
12289 default: return Register();
12290 }
12291}
12292
12293// FastEmit functions for ISD::SMAX.
12294
12295Register fastEmit_ISD_SMAX_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12296 if (RetVT.SimpleTy != MVT::i32)
12297 return Register();
12298 if ((Subtarget->hasCSSC())) {
12299 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12300 }
12301 return Register();
12302}
12303
12304Register fastEmit_ISD_SMAX_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12305 if (RetVT.SimpleTy != MVT::i64)
12306 return Register();
12307 if ((Subtarget->hasCSSC())) {
12308 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12309 }
12310 return Register();
12311}
12312
12313Register fastEmit_ISD_SMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12314 if (RetVT.SimpleTy != MVT::v8i8)
12315 return Register();
12316 if ((Subtarget->isNeonAvailable())) {
12317 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12318 }
12319 return Register();
12320}
12321
12322Register fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12323 if (RetVT.SimpleTy != MVT::v16i8)
12324 return Register();
12325 if ((Subtarget->isNeonAvailable())) {
12326 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12327 }
12328 return Register();
12329}
12330
12331Register fastEmit_ISD_SMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12332 if (RetVT.SimpleTy != MVT::v4i16)
12333 return Register();
12334 if ((Subtarget->isNeonAvailable())) {
12335 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12336 }
12337 return Register();
12338}
12339
12340Register fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12341 if (RetVT.SimpleTy != MVT::v8i16)
12342 return Register();
12343 if ((Subtarget->isNeonAvailable())) {
12344 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12345 }
12346 return Register();
12347}
12348
12349Register fastEmit_ISD_SMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12350 if (RetVT.SimpleTy != MVT::v2i32)
12351 return Register();
12352 if ((Subtarget->isNeonAvailable())) {
12353 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12354 }
12355 return Register();
12356}
12357
12358Register fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12359 if (RetVT.SimpleTy != MVT::v4i32)
12360 return Register();
12361 if ((Subtarget->isNeonAvailable())) {
12362 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12363 }
12364 return Register();
12365}
12366
12367Register fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12368 switch (VT.SimpleTy) {
12369 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_rr(RetVT, Op0, Op1);
12370 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_rr(RetVT, Op0, Op1);
12371 case MVT::v8i8: return fastEmit_ISD_SMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
12372 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
12373 case MVT::v4i16: return fastEmit_ISD_SMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
12374 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
12375 case MVT::v2i32: return fastEmit_ISD_SMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
12376 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
12377 default: return Register();
12378 }
12379}
12380
12381// FastEmit functions for ISD::SMIN.
12382
12383Register fastEmit_ISD_SMIN_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12384 if (RetVT.SimpleTy != MVT::i32)
12385 return Register();
12386 if ((Subtarget->hasCSSC())) {
12387 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12388 }
12389 return Register();
12390}
12391
12392Register fastEmit_ISD_SMIN_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12393 if (RetVT.SimpleTy != MVT::i64)
12394 return Register();
12395 if ((Subtarget->hasCSSC())) {
12396 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12397 }
12398 return Register();
12399}
12400
12401Register fastEmit_ISD_SMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12402 if (RetVT.SimpleTy != MVT::v8i8)
12403 return Register();
12404 if ((Subtarget->isNeonAvailable())) {
12405 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12406 }
12407 return Register();
12408}
12409
12410Register fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12411 if (RetVT.SimpleTy != MVT::v16i8)
12412 return Register();
12413 if ((Subtarget->isNeonAvailable())) {
12414 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12415 }
12416 return Register();
12417}
12418
12419Register fastEmit_ISD_SMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12420 if (RetVT.SimpleTy != MVT::v4i16)
12421 return Register();
12422 if ((Subtarget->isNeonAvailable())) {
12423 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12424 }
12425 return Register();
12426}
12427
12428Register fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12429 if (RetVT.SimpleTy != MVT::v8i16)
12430 return Register();
12431 if ((Subtarget->isNeonAvailable())) {
12432 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12433 }
12434 return Register();
12435}
12436
12437Register fastEmit_ISD_SMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12438 if (RetVT.SimpleTy != MVT::v2i32)
12439 return Register();
12440 if ((Subtarget->isNeonAvailable())) {
12441 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12442 }
12443 return Register();
12444}
12445
12446Register fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12447 if (RetVT.SimpleTy != MVT::v4i32)
12448 return Register();
12449 if ((Subtarget->isNeonAvailable())) {
12450 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12451 }
12452 return Register();
12453}
12454
12455Register fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12456 switch (VT.SimpleTy) {
12457 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_rr(RetVT, Op0, Op1);
12458 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_rr(RetVT, Op0, Op1);
12459 case MVT::v8i8: return fastEmit_ISD_SMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
12460 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
12461 case MVT::v4i16: return fastEmit_ISD_SMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
12462 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
12463 case MVT::v2i32: return fastEmit_ISD_SMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
12464 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
12465 default: return Register();
12466 }
12467}
12468
12469// FastEmit functions for ISD::SRA.
12470
12471Register fastEmit_ISD_SRA_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12472 if (RetVT.SimpleTy != MVT::i64)
12473 return Register();
12474 return fastEmitInst_rr(MachineInstOpcode: AArch64::ASRVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12475}
12476
12477Register fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12478 switch (VT.SimpleTy) {
12479 case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_rr(RetVT, Op0, Op1);
12480 default: return Register();
12481 }
12482}
12483
12484// FastEmit functions for ISD::SRL.
12485
12486Register fastEmit_ISD_SRL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12487 if (RetVT.SimpleTy != MVT::i64)
12488 return Register();
12489 return fastEmitInst_rr(MachineInstOpcode: AArch64::LSRVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12490}
12491
12492Register fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12493 switch (VT.SimpleTy) {
12494 case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_rr(RetVT, Op0, Op1);
12495 default: return Register();
12496 }
12497}
12498
12499// FastEmit functions for ISD::SSUBSAT.
12500
12501Register fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12502 if (RetVT.SimpleTy != MVT::v8i8)
12503 return Register();
12504 if ((Subtarget->isNeonAvailable())) {
12505 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12506 }
12507 return Register();
12508}
12509
12510Register fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12511 if (RetVT.SimpleTy != MVT::v16i8)
12512 return Register();
12513 if ((Subtarget->isNeonAvailable())) {
12514 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12515 }
12516 return Register();
12517}
12518
12519Register fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12520 if (RetVT.SimpleTy != MVT::v4i16)
12521 return Register();
12522 if ((Subtarget->isNeonAvailable())) {
12523 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12524 }
12525 return Register();
12526}
12527
12528Register fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12529 if (RetVT.SimpleTy != MVT::v8i16)
12530 return Register();
12531 if ((Subtarget->isNeonAvailable())) {
12532 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12533 }
12534 return Register();
12535}
12536
12537Register fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12538 if (RetVT.SimpleTy != MVT::v2i32)
12539 return Register();
12540 if ((Subtarget->isNeonAvailable())) {
12541 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12542 }
12543 return Register();
12544}
12545
12546Register fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12547 if (RetVT.SimpleTy != MVT::v4i32)
12548 return Register();
12549 if ((Subtarget->isNeonAvailable())) {
12550 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12551 }
12552 return Register();
12553}
12554
12555Register fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12556 if (RetVT.SimpleTy != MVT::v1i64)
12557 return Register();
12558 if ((Subtarget->isNeonAvailable())) {
12559 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
12560 }
12561 return Register();
12562}
12563
12564Register fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12565 if (RetVT.SimpleTy != MVT::v2i64)
12566 return Register();
12567 if ((Subtarget->isNeonAvailable())) {
12568 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12569 }
12570 return Register();
12571}
12572
12573Register fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12574 if (RetVT.SimpleTy != MVT::nxv16i8)
12575 return Register();
12576 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12577 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
12578 }
12579 return Register();
12580}
12581
12582Register fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12583 if (RetVT.SimpleTy != MVT::nxv8i16)
12584 return Register();
12585 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12586 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
12587 }
12588 return Register();
12589}
12590
12591Register fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12592 if (RetVT.SimpleTy != MVT::nxv4i32)
12593 return Register();
12594 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12595 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
12596 }
12597 return Register();
12598}
12599
12600Register fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12601 if (RetVT.SimpleTy != MVT::nxv2i64)
12602 return Register();
12603 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12604 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
12605 }
12606 return Register();
12607}
12608
12609Register fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12610 switch (VT.SimpleTy) {
12611 case MVT::v8i8: return fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
12612 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12613 case MVT::v4i16: return fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
12614 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12615 case MVT::v2i32: return fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
12616 case MVT::v4i32: return fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
12617 case MVT::v1i64: return fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
12618 case MVT::v2i64: return fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
12619 case MVT::nxv16i8: return fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12620 case MVT::nxv8i16: return fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12621 case MVT::nxv4i32: return fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12622 case MVT::nxv2i64: return fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12623 default: return Register();
12624 }
12625}
12626
12627// FastEmit functions for ISD::STRICT_FADD.
12628
12629Register fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12630 if (RetVT.SimpleTy != MVT::f16)
12631 return Register();
12632 if ((Subtarget->hasFullFP16())) {
12633 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12634 }
12635 return Register();
12636}
12637
12638Register fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12639 if (RetVT.SimpleTy != MVT::f32)
12640 return Register();
12641 if ((Subtarget->hasFPARMv8())) {
12642 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12643 }
12644 return Register();
12645}
12646
12647Register fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12648 if (RetVT.SimpleTy != MVT::f64)
12649 return Register();
12650 if ((Subtarget->hasFPARMv8())) {
12651 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12652 }
12653 return Register();
12654}
12655
12656Register fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12657 if (RetVT.SimpleTy != MVT::v4f16)
12658 return Register();
12659 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12660 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12661 }
12662 return Register();
12663}
12664
12665Register fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12666 if (RetVT.SimpleTy != MVT::v8f16)
12667 return Register();
12668 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12669 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12670 }
12671 return Register();
12672}
12673
12674Register fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12675 if (RetVT.SimpleTy != MVT::v2f32)
12676 return Register();
12677 if ((Subtarget->isNeonAvailable())) {
12678 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12679 }
12680 return Register();
12681}
12682
12683Register fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12684 if (RetVT.SimpleTy != MVT::v4f32)
12685 return Register();
12686 if ((Subtarget->isNeonAvailable())) {
12687 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12688 }
12689 return Register();
12690}
12691
12692Register fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12693 if (RetVT.SimpleTy != MVT::v2f64)
12694 return Register();
12695 if ((Subtarget->isNeonAvailable())) {
12696 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12697 }
12698 return Register();
12699}
12700
12701Register fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12702 switch (VT.SimpleTy) {
12703 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1);
12704 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1);
12705 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1);
12706 case MVT::v4f16: return fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
12707 case MVT::v8f16: return fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
12708 case MVT::v2f32: return fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
12709 case MVT::v4f32: return fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
12710 case MVT::v2f64: return fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
12711 default: return Register();
12712 }
12713}
12714
12715// FastEmit functions for ISD::STRICT_FDIV.
12716
12717Register fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12718 if (RetVT.SimpleTy != MVT::f16)
12719 return Register();
12720 if ((Subtarget->hasFullFP16())) {
12721 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12722 }
12723 return Register();
12724}
12725
12726Register fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12727 if (RetVT.SimpleTy != MVT::f32)
12728 return Register();
12729 if ((Subtarget->hasFPARMv8())) {
12730 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12731 }
12732 return Register();
12733}
12734
12735Register fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12736 if (RetVT.SimpleTy != MVT::f64)
12737 return Register();
12738 if ((Subtarget->hasFPARMv8())) {
12739 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12740 }
12741 return Register();
12742}
12743
12744Register fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12745 if (RetVT.SimpleTy != MVT::v4f16)
12746 return Register();
12747 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12748 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12749 }
12750 return Register();
12751}
12752
12753Register fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12754 if (RetVT.SimpleTy != MVT::v8f16)
12755 return Register();
12756 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12757 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12758 }
12759 return Register();
12760}
12761
12762Register fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12763 if (RetVT.SimpleTy != MVT::v2f32)
12764 return Register();
12765 if ((Subtarget->isNeonAvailable())) {
12766 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12767 }
12768 return Register();
12769}
12770
12771Register fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12772 if (RetVT.SimpleTy != MVT::v4f32)
12773 return Register();
12774 if ((Subtarget->isNeonAvailable())) {
12775 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12776 }
12777 return Register();
12778}
12779
12780Register fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12781 if (RetVT.SimpleTy != MVT::v2f64)
12782 return Register();
12783 if ((Subtarget->isNeonAvailable())) {
12784 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12785 }
12786 return Register();
12787}
12788
12789Register fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12790 switch (VT.SimpleTy) {
12791 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
12792 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
12793 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
12794 case MVT::v4f16: return fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1);
12795 case MVT::v8f16: return fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
12796 case MVT::v2f32: return fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1);
12797 case MVT::v4f32: return fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
12798 case MVT::v2f64: return fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
12799 default: return Register();
12800 }
12801}
12802
12803// FastEmit functions for ISD::STRICT_FMAXIMUM.
12804
12805Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12806 if (RetVT.SimpleTy != MVT::f16)
12807 return Register();
12808 if ((Subtarget->hasFullFP16())) {
12809 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12810 }
12811 return Register();
12812}
12813
12814Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12815 if (RetVT.SimpleTy != MVT::f32)
12816 return Register();
12817 if ((Subtarget->hasFPARMv8())) {
12818 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12819 }
12820 return Register();
12821}
12822
12823Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12824 if (RetVT.SimpleTy != MVT::f64)
12825 return Register();
12826 if ((Subtarget->hasFPARMv8())) {
12827 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12828 }
12829 return Register();
12830}
12831
12832Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12833 if (RetVT.SimpleTy != MVT::v4f16)
12834 return Register();
12835 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12836 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12837 }
12838 return Register();
12839}
12840
12841Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12842 if (RetVT.SimpleTy != MVT::v8f16)
12843 return Register();
12844 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12845 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12846 }
12847 return Register();
12848}
12849
12850Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12851 if (RetVT.SimpleTy != MVT::v2f32)
12852 return Register();
12853 if ((Subtarget->isNeonAvailable())) {
12854 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12855 }
12856 return Register();
12857}
12858
12859Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12860 if (RetVT.SimpleTy != MVT::v4f32)
12861 return Register();
12862 if ((Subtarget->isNeonAvailable())) {
12863 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12864 }
12865 return Register();
12866}
12867
12868Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12869 if (RetVT.SimpleTy != MVT::v2f64)
12870 return Register();
12871 if ((Subtarget->isNeonAvailable())) {
12872 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12873 }
12874 return Register();
12875}
12876
12877Register fastEmit_ISD_STRICT_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12878 switch (VT.SimpleTy) {
12879 case MVT::f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1);
12880 case MVT::f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1);
12881 case MVT::f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1);
12882 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
12883 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
12884 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
12885 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
12886 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
12887 default: return Register();
12888 }
12889}
12890
12891// FastEmit functions for ISD::STRICT_FMAXNUM.
12892
12893Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12894 if (RetVT.SimpleTy != MVT::f16)
12895 return Register();
12896 if ((Subtarget->hasFullFP16())) {
12897 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12898 }
12899 return Register();
12900}
12901
12902Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12903 if (RetVT.SimpleTy != MVT::f32)
12904 return Register();
12905 if ((Subtarget->hasFPARMv8())) {
12906 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12907 }
12908 return Register();
12909}
12910
12911Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12912 if (RetVT.SimpleTy != MVT::f64)
12913 return Register();
12914 if ((Subtarget->hasFPARMv8())) {
12915 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12916 }
12917 return Register();
12918}
12919
12920Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12921 if (RetVT.SimpleTy != MVT::v4f16)
12922 return Register();
12923 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12924 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12925 }
12926 return Register();
12927}
12928
12929Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12930 if (RetVT.SimpleTy != MVT::v8f16)
12931 return Register();
12932 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12933 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12934 }
12935 return Register();
12936}
12937
12938Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12939 if (RetVT.SimpleTy != MVT::v2f32)
12940 return Register();
12941 if ((Subtarget->isNeonAvailable())) {
12942 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12943 }
12944 return Register();
12945}
12946
12947Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12948 if (RetVT.SimpleTy != MVT::v4f32)
12949 return Register();
12950 if ((Subtarget->isNeonAvailable())) {
12951 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12952 }
12953 return Register();
12954}
12955
12956Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12957 if (RetVT.SimpleTy != MVT::v2f64)
12958 return Register();
12959 if ((Subtarget->isNeonAvailable())) {
12960 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12961 }
12962 return Register();
12963}
12964
12965Register fastEmit_ISD_STRICT_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12966 switch (VT.SimpleTy) {
12967 case MVT::f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
12968 case MVT::f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
12969 case MVT::f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
12970 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
12971 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
12972 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
12973 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
12974 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
12975 default: return Register();
12976 }
12977}
12978
12979// FastEmit functions for ISD::STRICT_FMINIMUM.
12980
12981Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12982 if (RetVT.SimpleTy != MVT::f16)
12983 return Register();
12984 if ((Subtarget->hasFullFP16())) {
12985 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12986 }
12987 return Register();
12988}
12989
12990Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12991 if (RetVT.SimpleTy != MVT::f32)
12992 return Register();
12993 if ((Subtarget->hasFPARMv8())) {
12994 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12995 }
12996 return Register();
12997}
12998
12999Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13000 if (RetVT.SimpleTy != MVT::f64)
13001 return Register();
13002 if ((Subtarget->hasFPARMv8())) {
13003 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13004 }
13005 return Register();
13006}
13007
13008Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13009 if (RetVT.SimpleTy != MVT::v4f16)
13010 return Register();
13011 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13012 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13013 }
13014 return Register();
13015}
13016
13017Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13018 if (RetVT.SimpleTy != MVT::v8f16)
13019 return Register();
13020 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13021 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13022 }
13023 return Register();
13024}
13025
13026Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13027 if (RetVT.SimpleTy != MVT::v2f32)
13028 return Register();
13029 if ((Subtarget->isNeonAvailable())) {
13030 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13031 }
13032 return Register();
13033}
13034
13035Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13036 if (RetVT.SimpleTy != MVT::v4f32)
13037 return Register();
13038 if ((Subtarget->isNeonAvailable())) {
13039 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13040 }
13041 return Register();
13042}
13043
13044Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13045 if (RetVT.SimpleTy != MVT::v2f64)
13046 return Register();
13047 if ((Subtarget->isNeonAvailable())) {
13048 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13049 }
13050 return Register();
13051}
13052
13053Register fastEmit_ISD_STRICT_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13054 switch (VT.SimpleTy) {
13055 case MVT::f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1);
13056 case MVT::f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1);
13057 case MVT::f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1);
13058 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13059 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13060 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13061 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13062 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13063 default: return Register();
13064 }
13065}
13066
13067// FastEmit functions for ISD::STRICT_FMINNUM.
13068
13069Register fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13070 if (RetVT.SimpleTy != MVT::f16)
13071 return Register();
13072 if ((Subtarget->hasFullFP16())) {
13073 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13074 }
13075 return Register();
13076}
13077
13078Register fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13079 if (RetVT.SimpleTy != MVT::f32)
13080 return Register();
13081 if ((Subtarget->hasFPARMv8())) {
13082 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13083 }
13084 return Register();
13085}
13086
13087Register fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13088 if (RetVT.SimpleTy != MVT::f64)
13089 return Register();
13090 if ((Subtarget->hasFPARMv8())) {
13091 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13092 }
13093 return Register();
13094}
13095
13096Register fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13097 if (RetVT.SimpleTy != MVT::v4f16)
13098 return Register();
13099 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13100 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13101 }
13102 return Register();
13103}
13104
13105Register fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13106 if (RetVT.SimpleTy != MVT::v8f16)
13107 return Register();
13108 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13109 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13110 }
13111 return Register();
13112}
13113
13114Register fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13115 if (RetVT.SimpleTy != MVT::v2f32)
13116 return Register();
13117 if ((Subtarget->isNeonAvailable())) {
13118 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13119 }
13120 return Register();
13121}
13122
13123Register fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13124 if (RetVT.SimpleTy != MVT::v4f32)
13125 return Register();
13126 if ((Subtarget->isNeonAvailable())) {
13127 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13128 }
13129 return Register();
13130}
13131
13132Register fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13133 if (RetVT.SimpleTy != MVT::v2f64)
13134 return Register();
13135 if ((Subtarget->isNeonAvailable())) {
13136 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13137 }
13138 return Register();
13139}
13140
13141Register fastEmit_ISD_STRICT_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13142 switch (VT.SimpleTy) {
13143 case MVT::f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
13144 case MVT::f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
13145 case MVT::f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
13146 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13147 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13148 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13149 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13150 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13151 default: return Register();
13152 }
13153}
13154
13155// FastEmit functions for ISD::STRICT_FMUL.
13156
13157Register fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13158 if (RetVT.SimpleTy != MVT::f16)
13159 return Register();
13160 if ((Subtarget->hasFullFP16())) {
13161 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13162 }
13163 return Register();
13164}
13165
13166Register fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13167 if (RetVT.SimpleTy != MVT::f32)
13168 return Register();
13169 if ((Subtarget->hasFPARMv8())) {
13170 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13171 }
13172 return Register();
13173}
13174
13175Register fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13176 if (RetVT.SimpleTy != MVT::f64)
13177 return Register();
13178 if ((Subtarget->hasFPARMv8())) {
13179 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13180 }
13181 return Register();
13182}
13183
13184Register fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13185 if (RetVT.SimpleTy != MVT::v4f16)
13186 return Register();
13187 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13188 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13189 }
13190 return Register();
13191}
13192
13193Register fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13194 if (RetVT.SimpleTy != MVT::v8f16)
13195 return Register();
13196 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13197 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13198 }
13199 return Register();
13200}
13201
13202Register fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13203 if (RetVT.SimpleTy != MVT::v2f32)
13204 return Register();
13205 if ((Subtarget->isNeonAvailable())) {
13206 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13207 }
13208 return Register();
13209}
13210
13211Register fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13212 if (RetVT.SimpleTy != MVT::v4f32)
13213 return Register();
13214 if ((Subtarget->isNeonAvailable())) {
13215 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13216 }
13217 return Register();
13218}
13219
13220Register fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13221 if (RetVT.SimpleTy != MVT::v2f64)
13222 return Register();
13223 if ((Subtarget->isNeonAvailable())) {
13224 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13225 }
13226 return Register();
13227}
13228
13229Register fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13230 switch (VT.SimpleTy) {
13231 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
13232 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
13233 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
13234 case MVT::v4f16: return fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
13235 case MVT::v8f16: return fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
13236 case MVT::v2f32: return fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
13237 case MVT::v4f32: return fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
13238 case MVT::v2f64: return fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
13239 default: return Register();
13240 }
13241}
13242
13243// FastEmit functions for ISD::STRICT_FSUB.
13244
13245Register fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13246 if (RetVT.SimpleTy != MVT::f16)
13247 return Register();
13248 if ((Subtarget->hasFullFP16())) {
13249 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13250 }
13251 return Register();
13252}
13253
13254Register fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13255 if (RetVT.SimpleTy != MVT::f32)
13256 return Register();
13257 if ((Subtarget->hasFPARMv8())) {
13258 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13259 }
13260 return Register();
13261}
13262
13263Register fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13264 if (RetVT.SimpleTy != MVT::f64)
13265 return Register();
13266 if ((Subtarget->hasFPARMv8())) {
13267 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13268 }
13269 return Register();
13270}
13271
13272Register fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13273 if (RetVT.SimpleTy != MVT::v4f16)
13274 return Register();
13275 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13276 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13277 }
13278 return Register();
13279}
13280
13281Register fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13282 if (RetVT.SimpleTy != MVT::v8f16)
13283 return Register();
13284 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13285 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13286 }
13287 return Register();
13288}
13289
13290Register fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13291 if (RetVT.SimpleTy != MVT::v2f32)
13292 return Register();
13293 if ((Subtarget->isNeonAvailable())) {
13294 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13295 }
13296 return Register();
13297}
13298
13299Register fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13300 if (RetVT.SimpleTy != MVT::v4f32)
13301 return Register();
13302 if ((Subtarget->isNeonAvailable())) {
13303 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13304 }
13305 return Register();
13306}
13307
13308Register fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13309 if (RetVT.SimpleTy != MVT::v2f64)
13310 return Register();
13311 if ((Subtarget->isNeonAvailable())) {
13312 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13313 }
13314 return Register();
13315}
13316
13317Register fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13318 switch (VT.SimpleTy) {
13319 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
13320 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
13321 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
13322 case MVT::v4f16: return fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
13323 case MVT::v8f16: return fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
13324 case MVT::v2f32: return fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
13325 case MVT::v4f32: return fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
13326 case MVT::v2f64: return fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
13327 default: return Register();
13328 }
13329}
13330
13331// FastEmit functions for ISD::SUB.
13332
13333Register fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13334 if (RetVT.SimpleTy != MVT::i32)
13335 return Register();
13336 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBSWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13337}
13338
13339Register fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13340 if (RetVT.SimpleTy != MVT::i64)
13341 return Register();
13342 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBSXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13343}
13344
13345Register fastEmit_ISD_SUB_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13346 if (RetVT.SimpleTy != MVT::v8i8)
13347 return Register();
13348 if ((Subtarget->isNeonAvailable())) {
13349 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13350 }
13351 return Register();
13352}
13353
13354Register fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13355 if (RetVT.SimpleTy != MVT::v16i8)
13356 return Register();
13357 if ((Subtarget->isNeonAvailable())) {
13358 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13359 }
13360 return Register();
13361}
13362
13363Register fastEmit_ISD_SUB_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13364 if (RetVT.SimpleTy != MVT::v4i16)
13365 return Register();
13366 if ((Subtarget->isNeonAvailable())) {
13367 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13368 }
13369 return Register();
13370}
13371
13372Register fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13373 if (RetVT.SimpleTy != MVT::v8i16)
13374 return Register();
13375 if ((Subtarget->isNeonAvailable())) {
13376 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13377 }
13378 return Register();
13379}
13380
13381Register fastEmit_ISD_SUB_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13382 if (RetVT.SimpleTy != MVT::v2i32)
13383 return Register();
13384 if ((Subtarget->isNeonAvailable())) {
13385 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13386 }
13387 return Register();
13388}
13389
13390Register fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13391 if (RetVT.SimpleTy != MVT::v4i32)
13392 return Register();
13393 if ((Subtarget->isNeonAvailable())) {
13394 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13395 }
13396 return Register();
13397}
13398
13399Register fastEmit_ISD_SUB_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13400 if (RetVT.SimpleTy != MVT::v1i64)
13401 return Register();
13402 if ((Subtarget->isNeonAvailable())) {
13403 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13404 }
13405 return Register();
13406}
13407
13408Register fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13409 if (RetVT.SimpleTy != MVT::v2i64)
13410 return Register();
13411 if ((Subtarget->isNeonAvailable())) {
13412 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13413 }
13414 return Register();
13415}
13416
13417Register fastEmit_ISD_SUB_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13418 if (RetVT.SimpleTy != MVT::nxv16i8)
13419 return Register();
13420 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13421 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13422 }
13423 return Register();
13424}
13425
13426Register fastEmit_ISD_SUB_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13427 if (RetVT.SimpleTy != MVT::nxv8i16)
13428 return Register();
13429 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13430 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13431 }
13432 return Register();
13433}
13434
13435Register fastEmit_ISD_SUB_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13436 if (RetVT.SimpleTy != MVT::nxv4i32)
13437 return Register();
13438 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13439 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13440 }
13441 return Register();
13442}
13443
13444Register fastEmit_ISD_SUB_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13445 if (RetVT.SimpleTy != MVT::nxv2i64)
13446 return Register();
13447 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13448 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13449 }
13450 return Register();
13451}
13452
13453Register fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13454 switch (VT.SimpleTy) {
13455 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
13456 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op1);
13457 case MVT::v8i8: return fastEmit_ISD_SUB_MVT_v8i8_rr(RetVT, Op0, Op1);
13458 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
13459 case MVT::v4i16: return fastEmit_ISD_SUB_MVT_v4i16_rr(RetVT, Op0, Op1);
13460 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
13461 case MVT::v2i32: return fastEmit_ISD_SUB_MVT_v2i32_rr(RetVT, Op0, Op1);
13462 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
13463 case MVT::v1i64: return fastEmit_ISD_SUB_MVT_v1i64_rr(RetVT, Op0, Op1);
13464 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
13465 case MVT::nxv16i8: return fastEmit_ISD_SUB_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13466 case MVT::nxv8i16: return fastEmit_ISD_SUB_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13467 case MVT::nxv4i32: return fastEmit_ISD_SUB_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13468 case MVT::nxv2i64: return fastEmit_ISD_SUB_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13469 default: return Register();
13470 }
13471}
13472
13473// FastEmit functions for ISD::UADDSAT.
13474
13475Register fastEmit_ISD_UADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13476 if (RetVT.SimpleTy != MVT::v8i8)
13477 return Register();
13478 if ((Subtarget->isNeonAvailable())) {
13479 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13480 }
13481 return Register();
13482}
13483
13484Register fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13485 if (RetVT.SimpleTy != MVT::v16i8)
13486 return Register();
13487 if ((Subtarget->isNeonAvailable())) {
13488 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13489 }
13490 return Register();
13491}
13492
13493Register fastEmit_ISD_UADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13494 if (RetVT.SimpleTy != MVT::v4i16)
13495 return Register();
13496 if ((Subtarget->isNeonAvailable())) {
13497 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13498 }
13499 return Register();
13500}
13501
13502Register fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13503 if (RetVT.SimpleTy != MVT::v8i16)
13504 return Register();
13505 if ((Subtarget->isNeonAvailable())) {
13506 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13507 }
13508 return Register();
13509}
13510
13511Register fastEmit_ISD_UADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13512 if (RetVT.SimpleTy != MVT::v2i32)
13513 return Register();
13514 if ((Subtarget->isNeonAvailable())) {
13515 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13516 }
13517 return Register();
13518}
13519
13520Register fastEmit_ISD_UADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13521 if (RetVT.SimpleTy != MVT::v4i32)
13522 return Register();
13523 if ((Subtarget->isNeonAvailable())) {
13524 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13525 }
13526 return Register();
13527}
13528
13529Register fastEmit_ISD_UADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13530 if (RetVT.SimpleTy != MVT::v1i64)
13531 return Register();
13532 if ((Subtarget->isNeonAvailable())) {
13533 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13534 }
13535 return Register();
13536}
13537
13538Register fastEmit_ISD_UADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13539 if (RetVT.SimpleTy != MVT::v2i64)
13540 return Register();
13541 if ((Subtarget->isNeonAvailable())) {
13542 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13543 }
13544 return Register();
13545}
13546
13547Register fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13548 if (RetVT.SimpleTy != MVT::nxv16i8)
13549 return Register();
13550 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13551 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13552 }
13553 return Register();
13554}
13555
13556Register fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13557 if (RetVT.SimpleTy != MVT::nxv8i16)
13558 return Register();
13559 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13560 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13561 }
13562 return Register();
13563}
13564
13565Register fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13566 if (RetVT.SimpleTy != MVT::nxv4i32)
13567 return Register();
13568 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13569 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13570 }
13571 return Register();
13572}
13573
13574Register fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13575 if (RetVT.SimpleTy != MVT::nxv2i64)
13576 return Register();
13577 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13578 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13579 }
13580 return Register();
13581}
13582
13583Register fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13584 switch (VT.SimpleTy) {
13585 case MVT::v8i8: return fastEmit_ISD_UADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
13586 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13587 case MVT::v4i16: return fastEmit_ISD_UADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
13588 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13589 case MVT::v2i32: return fastEmit_ISD_UADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
13590 case MVT::v4i32: return fastEmit_ISD_UADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
13591 case MVT::v1i64: return fastEmit_ISD_UADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
13592 case MVT::v2i64: return fastEmit_ISD_UADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
13593 case MVT::nxv16i8: return fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13594 case MVT::nxv8i16: return fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13595 case MVT::nxv4i32: return fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13596 case MVT::nxv2i64: return fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13597 default: return Register();
13598 }
13599}
13600
13601// FastEmit functions for ISD::UDIV.
13602
13603Register fastEmit_ISD_UDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13604 if (RetVT.SimpleTy != MVT::i32)
13605 return Register();
13606 return fastEmitInst_rr(MachineInstOpcode: AArch64::UDIVWr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13607}
13608
13609Register fastEmit_ISD_UDIV_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13610 if (RetVT.SimpleTy != MVT::i64)
13611 return Register();
13612 return fastEmitInst_rr(MachineInstOpcode: AArch64::UDIVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13613}
13614
13615Register fastEmit_ISD_UDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13616 switch (VT.SimpleTy) {
13617 case MVT::i32: return fastEmit_ISD_UDIV_MVT_i32_rr(RetVT, Op0, Op1);
13618 case MVT::i64: return fastEmit_ISD_UDIV_MVT_i64_rr(RetVT, Op0, Op1);
13619 default: return Register();
13620 }
13621}
13622
13623// FastEmit functions for ISD::UMAX.
13624
13625Register fastEmit_ISD_UMAX_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13626 if (RetVT.SimpleTy != MVT::i32)
13627 return Register();
13628 if ((Subtarget->hasCSSC())) {
13629 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13630 }
13631 return Register();
13632}
13633
13634Register fastEmit_ISD_UMAX_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13635 if (RetVT.SimpleTy != MVT::i64)
13636 return Register();
13637 if ((Subtarget->hasCSSC())) {
13638 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13639 }
13640 return Register();
13641}
13642
13643Register fastEmit_ISD_UMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13644 if (RetVT.SimpleTy != MVT::v8i8)
13645 return Register();
13646 if ((Subtarget->isNeonAvailable())) {
13647 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13648 }
13649 return Register();
13650}
13651
13652Register fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13653 if (RetVT.SimpleTy != MVT::v16i8)
13654 return Register();
13655 if ((Subtarget->isNeonAvailable())) {
13656 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13657 }
13658 return Register();
13659}
13660
13661Register fastEmit_ISD_UMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13662 if (RetVT.SimpleTy != MVT::v4i16)
13663 return Register();
13664 if ((Subtarget->isNeonAvailable())) {
13665 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13666 }
13667 return Register();
13668}
13669
13670Register fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13671 if (RetVT.SimpleTy != MVT::v8i16)
13672 return Register();
13673 if ((Subtarget->isNeonAvailable())) {
13674 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13675 }
13676 return Register();
13677}
13678
13679Register fastEmit_ISD_UMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13680 if (RetVT.SimpleTy != MVT::v2i32)
13681 return Register();
13682 if ((Subtarget->isNeonAvailable())) {
13683 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13684 }
13685 return Register();
13686}
13687
13688Register fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13689 if (RetVT.SimpleTy != MVT::v4i32)
13690 return Register();
13691 if ((Subtarget->isNeonAvailable())) {
13692 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13693 }
13694 return Register();
13695}
13696
13697Register fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13698 switch (VT.SimpleTy) {
13699 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_rr(RetVT, Op0, Op1);
13700 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_rr(RetVT, Op0, Op1);
13701 case MVT::v8i8: return fastEmit_ISD_UMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
13702 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
13703 case MVT::v4i16: return fastEmit_ISD_UMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
13704 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
13705 case MVT::v2i32: return fastEmit_ISD_UMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
13706 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
13707 default: return Register();
13708 }
13709}
13710
13711// FastEmit functions for ISD::UMIN.
13712
13713Register fastEmit_ISD_UMIN_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13714 if (RetVT.SimpleTy != MVT::i32)
13715 return Register();
13716 if ((Subtarget->hasCSSC())) {
13717 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13718 }
13719 return Register();
13720}
13721
13722Register fastEmit_ISD_UMIN_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13723 if (RetVT.SimpleTy != MVT::i64)
13724 return Register();
13725 if ((Subtarget->hasCSSC())) {
13726 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13727 }
13728 return Register();
13729}
13730
13731Register fastEmit_ISD_UMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13732 if (RetVT.SimpleTy != MVT::v8i8)
13733 return Register();
13734 if ((Subtarget->isNeonAvailable())) {
13735 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13736 }
13737 return Register();
13738}
13739
13740Register fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13741 if (RetVT.SimpleTy != MVT::v16i8)
13742 return Register();
13743 if ((Subtarget->isNeonAvailable())) {
13744 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13745 }
13746 return Register();
13747}
13748
13749Register fastEmit_ISD_UMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13750 if (RetVT.SimpleTy != MVT::v4i16)
13751 return Register();
13752 if ((Subtarget->isNeonAvailable())) {
13753 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13754 }
13755 return Register();
13756}
13757
13758Register fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13759 if (RetVT.SimpleTy != MVT::v8i16)
13760 return Register();
13761 if ((Subtarget->isNeonAvailable())) {
13762 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13763 }
13764 return Register();
13765}
13766
13767Register fastEmit_ISD_UMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13768 if (RetVT.SimpleTy != MVT::v2i32)
13769 return Register();
13770 if ((Subtarget->isNeonAvailable())) {
13771 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13772 }
13773 return Register();
13774}
13775
13776Register fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13777 if (RetVT.SimpleTy != MVT::v4i32)
13778 return Register();
13779 if ((Subtarget->isNeonAvailable())) {
13780 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13781 }
13782 return Register();
13783}
13784
13785Register fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13786 switch (VT.SimpleTy) {
13787 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_rr(RetVT, Op0, Op1);
13788 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_rr(RetVT, Op0, Op1);
13789 case MVT::v8i8: return fastEmit_ISD_UMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
13790 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
13791 case MVT::v4i16: return fastEmit_ISD_UMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
13792 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
13793 case MVT::v2i32: return fastEmit_ISD_UMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
13794 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
13795 default: return Register();
13796 }
13797}
13798
13799// FastEmit functions for ISD::USUBSAT.
13800
13801Register fastEmit_ISD_USUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13802 if (RetVT.SimpleTy != MVT::v8i8)
13803 return Register();
13804 if ((Subtarget->isNeonAvailable())) {
13805 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13806 }
13807 return Register();
13808}
13809
13810Register fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13811 if (RetVT.SimpleTy != MVT::v16i8)
13812 return Register();
13813 if ((Subtarget->isNeonAvailable())) {
13814 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13815 }
13816 return Register();
13817}
13818
13819Register fastEmit_ISD_USUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13820 if (RetVT.SimpleTy != MVT::v4i16)
13821 return Register();
13822 if ((Subtarget->isNeonAvailable())) {
13823 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13824 }
13825 return Register();
13826}
13827
13828Register fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13829 if (RetVT.SimpleTy != MVT::v8i16)
13830 return Register();
13831 if ((Subtarget->isNeonAvailable())) {
13832 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13833 }
13834 return Register();
13835}
13836
13837Register fastEmit_ISD_USUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13838 if (RetVT.SimpleTy != MVT::v2i32)
13839 return Register();
13840 if ((Subtarget->isNeonAvailable())) {
13841 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13842 }
13843 return Register();
13844}
13845
13846Register fastEmit_ISD_USUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13847 if (RetVT.SimpleTy != MVT::v4i32)
13848 return Register();
13849 if ((Subtarget->isNeonAvailable())) {
13850 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13851 }
13852 return Register();
13853}
13854
13855Register fastEmit_ISD_USUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13856 if (RetVT.SimpleTy != MVT::v1i64)
13857 return Register();
13858 if ((Subtarget->isNeonAvailable())) {
13859 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13860 }
13861 return Register();
13862}
13863
13864Register fastEmit_ISD_USUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13865 if (RetVT.SimpleTy != MVT::v2i64)
13866 return Register();
13867 if ((Subtarget->isNeonAvailable())) {
13868 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13869 }
13870 return Register();
13871}
13872
13873Register fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13874 if (RetVT.SimpleTy != MVT::nxv16i8)
13875 return Register();
13876 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13877 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13878 }
13879 return Register();
13880}
13881
13882Register fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13883 if (RetVT.SimpleTy != MVT::nxv8i16)
13884 return Register();
13885 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13886 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13887 }
13888 return Register();
13889}
13890
13891Register fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13892 if (RetVT.SimpleTy != MVT::nxv4i32)
13893 return Register();
13894 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13895 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13896 }
13897 return Register();
13898}
13899
13900Register fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13901 if (RetVT.SimpleTy != MVT::nxv2i64)
13902 return Register();
13903 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13904 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13905 }
13906 return Register();
13907}
13908
13909Register fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13910 switch (VT.SimpleTy) {
13911 case MVT::v8i8: return fastEmit_ISD_USUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
13912 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13913 case MVT::v4i16: return fastEmit_ISD_USUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
13914 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13915 case MVT::v2i32: return fastEmit_ISD_USUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
13916 case MVT::v4i32: return fastEmit_ISD_USUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
13917 case MVT::v1i64: return fastEmit_ISD_USUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
13918 case MVT::v2i64: return fastEmit_ISD_USUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
13919 case MVT::nxv16i8: return fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13920 case MVT::nxv8i16: return fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13921 case MVT::nxv4i32: return fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13922 case MVT::nxv2i64: return fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13923 default: return Register();
13924 }
13925}
13926
13927// FastEmit functions for ISD::XOR.
13928
13929Register fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13930 if (RetVT.SimpleTy != MVT::i32)
13931 return Register();
13932 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13933}
13934
13935Register fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13936 if (RetVT.SimpleTy != MVT::i64)
13937 return Register();
13938 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13939}
13940
13941Register fastEmit_ISD_XOR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13942 if (RetVT.SimpleTy != MVT::v8i8)
13943 return Register();
13944 if ((Subtarget->isNeonAvailable())) {
13945 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13946 }
13947 return Register();
13948}
13949
13950Register fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13951 if (RetVT.SimpleTy != MVT::v16i8)
13952 return Register();
13953 if ((Subtarget->isNeonAvailable())) {
13954 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13955 }
13956 return Register();
13957}
13958
13959Register fastEmit_ISD_XOR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13960 if (RetVT.SimpleTy != MVT::v4i16)
13961 return Register();
13962 if ((Subtarget->isNeonAvailable())) {
13963 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13964 }
13965 return Register();
13966}
13967
13968Register fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13969 if (RetVT.SimpleTy != MVT::v8i16)
13970 return Register();
13971 if ((Subtarget->isNeonAvailable())) {
13972 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13973 }
13974 return Register();
13975}
13976
13977Register fastEmit_ISD_XOR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13978 if (RetVT.SimpleTy != MVT::v2i32)
13979 return Register();
13980 if ((Subtarget->isNeonAvailable())) {
13981 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13982 }
13983 return Register();
13984}
13985
13986Register fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13987 if (RetVT.SimpleTy != MVT::v4i32)
13988 return Register();
13989 if ((Subtarget->isNeonAvailable())) {
13990 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13991 }
13992 return Register();
13993}
13994
13995Register fastEmit_ISD_XOR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13996 if (RetVT.SimpleTy != MVT::v1i64)
13997 return Register();
13998 if ((Subtarget->isNeonAvailable())) {
13999 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14000 }
14001 return Register();
14002}
14003
14004Register fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14005 if (RetVT.SimpleTy != MVT::v2i64)
14006 return Register();
14007 if ((Subtarget->isNeonAvailable())) {
14008 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14009 }
14010 return Register();
14011}
14012
14013Register fastEmit_ISD_XOR_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14014 if (RetVT.SimpleTy != MVT::nxv16i8)
14015 return Register();
14016 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14017 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14018 }
14019 return Register();
14020}
14021
14022Register fastEmit_ISD_XOR_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14023 if (RetVT.SimpleTy != MVT::nxv8i16)
14024 return Register();
14025 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14026 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14027 }
14028 return Register();
14029}
14030
14031Register fastEmit_ISD_XOR_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14032 if (RetVT.SimpleTy != MVT::nxv4i32)
14033 return Register();
14034 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14035 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14036 }
14037 return Register();
14038}
14039
14040Register fastEmit_ISD_XOR_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14041 if (RetVT.SimpleTy != MVT::nxv2i64)
14042 return Register();
14043 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14044 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14045 }
14046 return Register();
14047}
14048
14049Register fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14050 switch (VT.SimpleTy) {
14051 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
14052 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op1);
14053 case MVT::v8i8: return fastEmit_ISD_XOR_MVT_v8i8_rr(RetVT, Op0, Op1);
14054 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
14055 case MVT::v4i16: return fastEmit_ISD_XOR_MVT_v4i16_rr(RetVT, Op0, Op1);
14056 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
14057 case MVT::v2i32: return fastEmit_ISD_XOR_MVT_v2i32_rr(RetVT, Op0, Op1);
14058 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
14059 case MVT::v1i64: return fastEmit_ISD_XOR_MVT_v1i64_rr(RetVT, Op0, Op1);
14060 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
14061 case MVT::nxv16i8: return fastEmit_ISD_XOR_MVT_nxv16i8_rr(RetVT, Op0, Op1);
14062 case MVT::nxv8i16: return fastEmit_ISD_XOR_MVT_nxv8i16_rr(RetVT, Op0, Op1);
14063 case MVT::nxv4i32: return fastEmit_ISD_XOR_MVT_nxv4i32_rr(RetVT, Op0, Op1);
14064 case MVT::nxv2i64: return fastEmit_ISD_XOR_MVT_nxv2i64_rr(RetVT, Op0, Op1);
14065 default: return Register();
14066 }
14067}
14068
14069// Top-level FastEmit function.
14070
14071Register fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, Register Op1) override {
14072 switch (Opcode) {
14073 case AArch64ISD::ADDHN: return fastEmit_AArch64ISD_ADDHN_rr(VT, RetVT, Op0, Op1);
14074 case AArch64ISD::ADDP: return fastEmit_AArch64ISD_ADDP_rr(VT, RetVT, Op0, Op1);
14075 case AArch64ISD::BIC: return fastEmit_AArch64ISD_BIC_rr(VT, RetVT, Op0, Op1);
14076 case AArch64ISD::BRKB_Z: return fastEmit_AArch64ISD_BRKB_Z_rr(VT, RetVT, Op0, Op1);
14077 case AArch64ISD::CMTST: return fastEmit_AArch64ISD_CMTST_rr(VT, RetVT, Op0, Op1);
14078 case AArch64ISD::CTTZ_ELTS_ZERO_POISON: return fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_rr(VT, RetVT, Op0, Op1);
14079 case AArch64ISD::FCMEQ: return fastEmit_AArch64ISD_FCMEQ_rr(VT, RetVT, Op0, Op1);
14080 case AArch64ISD::FCMGE: return fastEmit_AArch64ISD_FCMGE_rr(VT, RetVT, Op0, Op1);
14081 case AArch64ISD::FCMGT: return fastEmit_AArch64ISD_FCMGT_rr(VT, RetVT, Op0, Op1);
14082 case AArch64ISD::FRECPS: return fastEmit_AArch64ISD_FRECPS_rr(VT, RetVT, Op0, Op1);
14083 case AArch64ISD::FRSQRTS: return fastEmit_AArch64ISD_FRSQRTS_rr(VT, RetVT, Op0, Op1);
14084 case AArch64ISD::PMULL: return fastEmit_AArch64ISD_PMULL_rr(VT, RetVT, Op0, Op1);
14085 case AArch64ISD::SMULL: return fastEmit_AArch64ISD_SMULL_rr(VT, RetVT, Op0, Op1);
14086 case AArch64ISD::SQADD: return fastEmit_AArch64ISD_SQADD_rr(VT, RetVT, Op0, Op1);
14087 case AArch64ISD::SQDMULH: return fastEmit_AArch64ISD_SQDMULH_rr(VT, RetVT, Op0, Op1);
14088 case AArch64ISD::SQDMULL: return fastEmit_AArch64ISD_SQDMULL_rr(VT, RetVT, Op0, Op1);
14089 case AArch64ISD::SQRDMULH: return fastEmit_AArch64ISD_SQRDMULH_rr(VT, RetVT, Op0, Op1);
14090 case AArch64ISD::SQRSHL: return fastEmit_AArch64ISD_SQRSHL_rr(VT, RetVT, Op0, Op1);
14091 case AArch64ISD::SQSHL: return fastEmit_AArch64ISD_SQSHL_rr(VT, RetVT, Op0, Op1);
14092 case AArch64ISD::SQSUB: return fastEmit_AArch64ISD_SQSUB_rr(VT, RetVT, Op0, Op1);
14093 case AArch64ISD::SUQADD: return fastEmit_AArch64ISD_SUQADD_rr(VT, RetVT, Op0, Op1);
14094 case AArch64ISD::TBL: return fastEmit_AArch64ISD_TBL_rr(VT, RetVT, Op0, Op1);
14095 case AArch64ISD::TRN1: return fastEmit_AArch64ISD_TRN1_rr(VT, RetVT, Op0, Op1);
14096 case AArch64ISD::TRN2: return fastEmit_AArch64ISD_TRN2_rr(VT, RetVT, Op0, Op1);
14097 case AArch64ISD::UMULL: return fastEmit_AArch64ISD_UMULL_rr(VT, RetVT, Op0, Op1);
14098 case AArch64ISD::UQADD: return fastEmit_AArch64ISD_UQADD_rr(VT, RetVT, Op0, Op1);
14099 case AArch64ISD::UQRSHL: return fastEmit_AArch64ISD_UQRSHL_rr(VT, RetVT, Op0, Op1);
14100 case AArch64ISD::UQSHL: return fastEmit_AArch64ISD_UQSHL_rr(VT, RetVT, Op0, Op1);
14101 case AArch64ISD::UQSUB: return fastEmit_AArch64ISD_UQSUB_rr(VT, RetVT, Op0, Op1);
14102 case AArch64ISD::USQADD: return fastEmit_AArch64ISD_USQADD_rr(VT, RetVT, Op0, Op1);
14103 case AArch64ISD::UZP1: return fastEmit_AArch64ISD_UZP1_rr(VT, RetVT, Op0, Op1);
14104 case AArch64ISD::UZP2: return fastEmit_AArch64ISD_UZP2_rr(VT, RetVT, Op0, Op1);
14105 case AArch64ISD::ZIP1: return fastEmit_AArch64ISD_ZIP1_rr(VT, RetVT, Op0, Op1);
14106 case AArch64ISD::ZIP2: return fastEmit_AArch64ISD_ZIP2_rr(VT, RetVT, Op0, Op1);
14107 case ISD::ABDS: return fastEmit_ISD_ABDS_rr(VT, RetVT, Op0, Op1);
14108 case ISD::ABDU: return fastEmit_ISD_ABDU_rr(VT, RetVT, Op0, Op1);
14109 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
14110 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
14111 case ISD::AVGCEILS: return fastEmit_ISD_AVGCEILS_rr(VT, RetVT, Op0, Op1);
14112 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
14113 case ISD::AVGFLOORS: return fastEmit_ISD_AVGFLOORS_rr(VT, RetVT, Op0, Op1);
14114 case ISD::AVGFLOORU: return fastEmit_ISD_AVGFLOORU_rr(VT, RetVT, Op0, Op1);
14115 case ISD::CLMUL: return fastEmit_ISD_CLMUL_rr(VT, RetVT, Op0, Op1);
14116 case ISD::CONCAT_VECTORS: return fastEmit_ISD_CONCAT_VECTORS_rr(VT, RetVT, Op0, Op1);
14117 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
14118 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
14119 case ISD::FMAXIMUM: return fastEmit_ISD_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
14120 case ISD::FMAXNUM: return fastEmit_ISD_FMAXNUM_rr(VT, RetVT, Op0, Op1);
14121 case ISD::FMAXNUM_IEEE: return fastEmit_ISD_FMAXNUM_IEEE_rr(VT, RetVT, Op0, Op1);
14122 case ISD::FMINIMUM: return fastEmit_ISD_FMINIMUM_rr(VT, RetVT, Op0, Op1);
14123 case ISD::FMINNUM: return fastEmit_ISD_FMINNUM_rr(VT, RetVT, Op0, Op1);
14124 case ISD::FMINNUM_IEEE: return fastEmit_ISD_FMINNUM_IEEE_rr(VT, RetVT, Op0, Op1);
14125 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
14126 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
14127 case ISD::GET_ACTIVE_LANE_MASK: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_rr(VT, RetVT, Op0, Op1);
14128 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
14129 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
14130 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
14131 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
14132 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
14133 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
14134 case ISD::SDIV: return fastEmit_ISD_SDIV_rr(VT, RetVT, Op0, Op1);
14135 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
14136 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
14137 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
14138 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
14139 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
14140 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
14141 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1);
14142 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1);
14143 case ISD::STRICT_FMAXIMUM: return fastEmit_ISD_STRICT_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
14144 case ISD::STRICT_FMAXNUM: return fastEmit_ISD_STRICT_FMAXNUM_rr(VT, RetVT, Op0, Op1);
14145 case ISD::STRICT_FMINIMUM: return fastEmit_ISD_STRICT_FMINIMUM_rr(VT, RetVT, Op0, Op1);
14146 case ISD::STRICT_FMINNUM: return fastEmit_ISD_STRICT_FMINNUM_rr(VT, RetVT, Op0, Op1);
14147 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1);
14148 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1);
14149 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
14150 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
14151 case ISD::UDIV: return fastEmit_ISD_UDIV_rr(VT, RetVT, Op0, Op1);
14152 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
14153 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
14154 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
14155 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
14156 default: return Register();
14157 }
14158}
14159
14160// FastEmit functions for AArch64ISD::DUPLANE64.
14161
14162Register fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14163 if (RetVT.SimpleTy != MVT::v2i64)
14164 return Register();
14165 if ((Subtarget->isNeonAvailable())) {
14166 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i64lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14167 }
14168 return Register();
14169}
14170
14171Register fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14172 if (RetVT.SimpleTy != MVT::v2f64)
14173 return Register();
14174 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i64lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14175}
14176
14177Register fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14178 switch (VT.SimpleTy) {
14179 case MVT::v2i64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14180 case MVT::v2f64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14181 default: return Register();
14182 }
14183}
14184
14185// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14186
14187Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14188 if (RetVT.SimpleTy != MVT::i64)
14189 return Register();
14190 if ((Subtarget->isNeonAvailable())) {
14191 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi64, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14192 }
14193 return Register();
14194}
14195
14196Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14197 if (RetVT.SimpleTy != MVT::f64)
14198 return Register();
14199 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi64, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14200}
14201
14202Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14203 switch (VT.SimpleTy) {
14204 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14205 case MVT::v2f64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14206 default: return Register();
14207 }
14208}
14209
14210// Top-level FastEmit function.
14211
14212Register fastEmit_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14213 switch (Opcode) {
14214 case AArch64ISD::DUPLANE64: return fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1);
14215 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1);
14216 default: return Register();
14217 }
14218}
14219
14220// FastEmit functions for AArch64ISD::DUPLANE32.
14221
14222Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14223 if ((Subtarget->isNeonAvailable())) {
14224 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i32lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14225 }
14226 return Register();
14227}
14228
14229Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14230 if ((Subtarget->isNeonAvailable())) {
14231 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i32lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14232 }
14233 return Register();
14234}
14235
14236Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14237switch (RetVT.SimpleTy) {
14238 case MVT::v2i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Op0, imm1);
14239 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Op0, imm1);
14240 default: return Register();
14241}
14242}
14243
14244Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14245 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i32lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14246}
14247
14248Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14249 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i32lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14250}
14251
14252Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14253switch (RetVT.SimpleTy) {
14254 case MVT::v2f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Op0, imm1);
14255 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Op0, imm1);
14256 default: return Register();
14257}
14258}
14259
14260Register fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14261 switch (VT.SimpleTy) {
14262 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14263 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14264 default: return Register();
14265 }
14266}
14267
14268// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14269
14270Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14271 if (RetVT.SimpleTy != MVT::i32)
14272 return Register();
14273 if ((Subtarget->isNeonAvailable())) {
14274 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi32, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14275 }
14276 return Register();
14277}
14278
14279Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14280 if (RetVT.SimpleTy != MVT::f32)
14281 return Register();
14282 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi32, RC: &AArch64::FPR32RegClass, Op0, Imm: imm1);
14283}
14284
14285Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14286 switch (VT.SimpleTy) {
14287 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14288 case MVT::v4f32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14289 default: return Register();
14290 }
14291}
14292
14293// Top-level FastEmit function.
14294
14295Register fastEmit_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14296 switch (Opcode) {
14297 case AArch64ISD::DUPLANE32: return fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1);
14298 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1);
14299 default: return Register();
14300 }
14301}
14302
14303// FastEmit functions for AArch64ISD::DUPLANE16.
14304
14305Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14306 if ((Subtarget->isNeonAvailable())) {
14307 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14308 }
14309 return Register();
14310}
14311
14312Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14313 if ((Subtarget->isNeonAvailable())) {
14314 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14315 }
14316 return Register();
14317}
14318
14319Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14320switch (RetVT.SimpleTy) {
14321 case MVT::v4i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Op0, imm1);
14322 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Op0, imm1);
14323 default: return Register();
14324}
14325}
14326
14327Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14328 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14329}
14330
14331Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14332 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14333}
14334
14335Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14336switch (RetVT.SimpleTy) {
14337 case MVT::v4f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Op0, imm1);
14338 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Op0, imm1);
14339 default: return Register();
14340}
14341}
14342
14343Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14344 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14345}
14346
14347Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14348 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14349}
14350
14351Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14352switch (RetVT.SimpleTy) {
14353 case MVT::v4bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(Op0, imm1);
14354 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(Op0, imm1);
14355 default: return Register();
14356}
14357}
14358
14359Register fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14360 switch (VT.SimpleTy) {
14361 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14362 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14363 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14364 default: return Register();
14365 }
14366}
14367
14368// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14369
14370Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14371 if (RetVT.SimpleTy != MVT::i32)
14372 return Register();
14373 if ((Subtarget->isNeonAvailable())) {
14374 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi16, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14375 }
14376 return Register();
14377}
14378
14379Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14380 if (RetVT.SimpleTy != MVT::f16)
14381 return Register();
14382 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi16, RC: &AArch64::FPR16RegClass, Op0, Imm: imm1);
14383}
14384
14385Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14386 if (RetVT.SimpleTy != MVT::bf16)
14387 return Register();
14388 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi16, RC: &AArch64::FPR16RegClass, Op0, Imm: imm1);
14389}
14390
14391Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14392 switch (VT.SimpleTy) {
14393 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14394 case MVT::v8f16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14395 case MVT::v8bf16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14396 default: return Register();
14397 }
14398}
14399
14400// Top-level FastEmit function.
14401
14402Register fastEmit_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14403 switch (Opcode) {
14404 case AArch64ISD::DUPLANE16: return fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1);
14405 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1);
14406 default: return Register();
14407 }
14408}
14409
14410// FastEmit functions for AArch64ISD::DUPLANE8.
14411
14412Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Register Op0, uint64_t imm1) {
14413 if ((Subtarget->isNeonAvailable())) {
14414 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i8lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14415 }
14416 return Register();
14417}
14418
14419Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Register Op0, uint64_t imm1) {
14420 if ((Subtarget->isNeonAvailable())) {
14421 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv16i8lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14422 }
14423 return Register();
14424}
14425
14426Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, Register Op0, uint64_t imm1) {
14427switch (RetVT.SimpleTy) {
14428 case MVT::v8i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Op0, imm1);
14429 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Op0, imm1);
14430 default: return Register();
14431}
14432}
14433
14434Register fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14435 switch (VT.SimpleTy) {
14436 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1);
14437 default: return Register();
14438 }
14439}
14440
14441// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14442
14443Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, Register Op0, uint64_t imm1) {
14444 if (RetVT.SimpleTy != MVT::i32)
14445 return Register();
14446 if ((Subtarget->isNeonAvailable())) {
14447 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi8, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14448 }
14449 return Register();
14450}
14451
14452Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14453 switch (VT.SimpleTy) {
14454 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1);
14455 default: return Register();
14456 }
14457}
14458
14459// Top-level FastEmit function.
14460
14461Register fastEmit_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14462 switch (Opcode) {
14463 case AArch64ISD::DUPLANE8: return fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1);
14464 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1);
14465 default: return Register();
14466 }
14467}
14468
14469// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14470
14471Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14472 if (RetVT.SimpleTy != MVT::i32)
14473 return Register();
14474 if ((Subtarget->hasNEON())) {
14475 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi8_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14476 }
14477 return Register();
14478}
14479
14480Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14481 if (RetVT.SimpleTy != MVT::i32)
14482 return Register();
14483 if ((Subtarget->hasNEON())) {
14484 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi16_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14485 }
14486 return Register();
14487}
14488
14489Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14490 if (RetVT.SimpleTy != MVT::i32)
14491 return Register();
14492 if ((Subtarget->hasNEON())) {
14493 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi32_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14494 }
14495 return Register();
14496}
14497
14498Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14499 if (RetVT.SimpleTy != MVT::i64)
14500 return Register();
14501 if ((Subtarget->hasNEON())) {
14502 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi64_idx0, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14503 }
14504 return Register();
14505}
14506
14507Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14508 switch (VT.SimpleTy) {
14509 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14510 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14511 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14512 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14513 default: return Register();
14514 }
14515}
14516
14517// Top-level FastEmit function.
14518
14519Register fastEmit_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14520 switch (Opcode) {
14521 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(VT, RetVT, Op0, imm1);
14522 default: return Register();
14523 }
14524}
14525
14526// FastEmit functions for ISD::SMAX.
14527
14528Register fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14529 if (RetVT.SimpleTy != MVT::i32)
14530 return Register();
14531 if ((Subtarget->hasCSSC())) {
14532 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMAXWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14533 }
14534 return Register();
14535}
14536
14537Register fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14538 switch (VT.SimpleTy) {
14539 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1);
14540 default: return Register();
14541 }
14542}
14543
14544// FastEmit functions for ISD::SMIN.
14545
14546Register fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14547 if (RetVT.SimpleTy != MVT::i32)
14548 return Register();
14549 if ((Subtarget->hasCSSC())) {
14550 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMINWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14551 }
14552 return Register();
14553}
14554
14555Register fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14556 switch (VT.SimpleTy) {
14557 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1);
14558 default: return Register();
14559 }
14560}
14561
14562// Top-level FastEmit function.
14563
14564Register fastEmit_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14565 switch (Opcode) {
14566 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1);
14567 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1);
14568 default: return Register();
14569 }
14570}
14571
14572// FastEmit functions for ISD::SMAX.
14573
14574Register fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14575 if (RetVT.SimpleTy != MVT::i64)
14576 return Register();
14577 if ((Subtarget->hasCSSC())) {
14578 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMAXXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14579 }
14580 return Register();
14581}
14582
14583Register fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14584 switch (VT.SimpleTy) {
14585 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1);
14586 default: return Register();
14587 }
14588}
14589
14590// FastEmit functions for ISD::SMIN.
14591
14592Register fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14593 if (RetVT.SimpleTy != MVT::i64)
14594 return Register();
14595 if ((Subtarget->hasCSSC())) {
14596 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMINXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14597 }
14598 return Register();
14599}
14600
14601Register fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14602 switch (VT.SimpleTy) {
14603 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1);
14604 default: return Register();
14605 }
14606}
14607
14608// Top-level FastEmit function.
14609
14610Register fastEmit_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14611 switch (Opcode) {
14612 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1);
14613 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1);
14614 default: return Register();
14615 }
14616}
14617
14618// FastEmit functions for ISD::UMAX.
14619
14620Register fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14621 if (RetVT.SimpleTy != MVT::i32)
14622 return Register();
14623 if ((Subtarget->hasCSSC())) {
14624 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMAXWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14625 }
14626 return Register();
14627}
14628
14629Register fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14630 switch (VT.SimpleTy) {
14631 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1);
14632 default: return Register();
14633 }
14634}
14635
14636// FastEmit functions for ISD::UMIN.
14637
14638Register fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14639 if (RetVT.SimpleTy != MVT::i32)
14640 return Register();
14641 if ((Subtarget->hasCSSC())) {
14642 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMINWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14643 }
14644 return Register();
14645}
14646
14647Register fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14648 switch (VT.SimpleTy) {
14649 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1);
14650 default: return Register();
14651 }
14652}
14653
14654// Top-level FastEmit function.
14655
14656Register fastEmit_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14657 switch (Opcode) {
14658 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1);
14659 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1);
14660 default: return Register();
14661 }
14662}
14663
14664// FastEmit functions for ISD::UMAX.
14665
14666Register fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14667 if (RetVT.SimpleTy != MVT::i64)
14668 return Register();
14669 if ((Subtarget->hasCSSC())) {
14670 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMAXXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14671 }
14672 return Register();
14673}
14674
14675Register fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14676 switch (VT.SimpleTy) {
14677 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1);
14678 default: return Register();
14679 }
14680}
14681
14682// FastEmit functions for ISD::UMIN.
14683
14684Register fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14685 if (RetVT.SimpleTy != MVT::i64)
14686 return Register();
14687 if ((Subtarget->hasCSSC())) {
14688 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMINXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14689 }
14690 return Register();
14691}
14692
14693Register fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14694 switch (VT.SimpleTy) {
14695 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1);
14696 default: return Register();
14697 }
14698}
14699
14700// Top-level FastEmit function.
14701
14702Register fastEmit_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14703 switch (Opcode) {
14704 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1);
14705 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1);
14706 default: return Register();
14707 }
14708}
14709
14710// FastEmit functions for ISD::Constant.
14711
14712Register fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
14713 if (RetVT.SimpleTy != MVT::i32)
14714 return Register();
14715 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVi32imm, RC: &AArch64::GPR32RegClass, Imm: imm0);
14716}
14717
14718Register fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) {
14719 if (RetVT.SimpleTy != MVT::i64)
14720 return Register();
14721 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVi64imm, RC: &AArch64::GPR64RegClass, Imm: imm0);
14722}
14723
14724Register fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
14725 switch (VT.SimpleTy) {
14726 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
14727 case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0);
14728 default: return Register();
14729 }
14730}
14731
14732// Top-level FastEmit function.
14733
14734Register fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
14735 if (VT == MVT::i32 && Predicate_imm0_255(Imm: imm0))
14736 if (Register Reg = fastEmit_i_Predicate_imm0_255(VT, RetVT, Opcode, imm0))
14737 return Reg;
14738
14739 if (VT == MVT::i32 && Predicate_simm6_32b(Imm: imm0))
14740 if (Register Reg = fastEmit_i_Predicate_simm6_32b(VT, RetVT, Opcode, imm0))
14741 return Reg;
14742
14743 switch (Opcode) {
14744 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
14745 default: return Register();
14746 }
14747}
14748
14749// FastEmit functions for AArch64ISD::FMOV.
14750
14751Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(uint64_t imm0) {
14752 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
14753 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv4f16_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
14754 }
14755 return Register();
14756}
14757
14758Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(uint64_t imm0) {
14759 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
14760 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv8f16_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14761 }
14762 return Register();
14763}
14764
14765Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(uint64_t imm0) {
14766 if ((Subtarget->isNeonAvailable())) {
14767 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv2f32_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
14768 }
14769 return Register();
14770}
14771
14772Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(uint64_t imm0) {
14773 if ((Subtarget->isNeonAvailable())) {
14774 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv4f32_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14775 }
14776 return Register();
14777}
14778
14779Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(uint64_t imm0) {
14780 if ((Subtarget->isNeonAvailable())) {
14781 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv2f64_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14782 }
14783 return Register();
14784}
14785
14786Register fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
14787switch (RetVT.SimpleTy) {
14788 case MVT::v4f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(imm0);
14789 case MVT::v8f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(imm0);
14790 case MVT::v2f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(imm0);
14791 case MVT::v4f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(imm0);
14792 case MVT::v2f64: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(imm0);
14793 default: return Register();
14794}
14795}
14796
14797Register fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
14798 switch (VT.SimpleTy) {
14799 case MVT::i32: return fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
14800 default: return Register();
14801 }
14802}
14803
14804// FastEmit functions for AArch64ISD::MOVI.
14805
14806Register fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(uint64_t imm0) {
14807 if ((Subtarget->isNeonAvailable())) {
14808 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv8b_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
14809 }
14810 return Register();
14811}
14812
14813Register fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(uint64_t imm0) {
14814 if ((Subtarget->isNeonAvailable())) {
14815 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv16b_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14816 }
14817 return Register();
14818}
14819
14820Register fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
14821switch (RetVT.SimpleTy) {
14822 case MVT::v8i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(imm0);
14823 case MVT::v16i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(imm0);
14824 default: return Register();
14825}
14826}
14827
14828Register fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
14829 switch (VT.SimpleTy) {
14830 case MVT::i32: return fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
14831 default: return Register();
14832 }
14833}
14834
14835// FastEmit functions for AArch64ISD::MOVIedit.
14836
14837Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(uint64_t imm0) {
14838 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVID, RC: &AArch64::FPR64RegClass, Imm: imm0);
14839}
14840
14841Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(uint64_t imm0) {
14842 if ((Subtarget->isNeonAvailable())) {
14843 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv2d_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14844 }
14845 return Register();
14846}
14847
14848Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
14849switch (RetVT.SimpleTy) {
14850 case MVT::f64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(imm0);
14851 case MVT::v2i64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(imm0);
14852 default: return Register();
14853}
14854}
14855
14856Register fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
14857 switch (VT.SimpleTy) {
14858 case MVT::i32: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
14859 default: return Register();
14860 }
14861}
14862
14863// Top-level FastEmit function.
14864
14865Register fastEmit_i_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
14866 switch (Opcode) {
14867 case AArch64ISD::FMOV: return fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(VT, RetVT, imm0);
14868 case AArch64ISD::MOVI: return fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(VT, RetVT, imm0);
14869 case AArch64ISD::MOVIedit: return fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(VT, RetVT, imm0);
14870 default: return Register();
14871 }
14872}
14873
14874// FastEmit functions for AArch64ISD::RDSVL.
14875
14876Register fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(MVT RetVT, uint64_t imm0) {
14877 if (RetVT.SimpleTy != MVT::i64)
14878 return Register();
14879 if ((Subtarget->hasSME())) {
14880 return fastEmitInst_i(MachineInstOpcode: AArch64::RDSVLI_XI, RC: &AArch64::GPR64RegClass, Imm: imm0);
14881 }
14882 return Register();
14883}
14884
14885Register fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(MVT VT, MVT RetVT, uint64_t imm0) {
14886 switch (VT.SimpleTy) {
14887 case MVT::i32: return fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(RetVT, imm0);
14888 default: return Register();
14889 }
14890}
14891
14892// Top-level FastEmit function.
14893
14894Register fastEmit_i_Predicate_simm6_32b(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
14895 switch (Opcode) {
14896 case AArch64ISD::RDSVL: return fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(VT, RetVT, imm0);
14897 default: return Register();
14898 }
14899}
14900
14901