1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the AArch64 target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_imm0_31(int64_t Imm) {
12
13 return ((uint64_t)Imm) < 32;
14
15}
16static bool Predicate_imm0_63(int64_t Imm) {
17
18 return ((uint64_t)Imm) < 64;
19
20}
21static bool Predicate_VectorIndexD(int64_t Imm) {
22 return ((uint64_t)Imm) < 2;
23}
24static bool Predicate_VectorIndexS(int64_t Imm) {
25 return ((uint64_t)Imm) < 4;
26}
27static bool Predicate_VectorIndexH(int64_t Imm) {
28 return ((uint64_t)Imm) < 8;
29}
30static bool Predicate_VectorIndexB(int64_t Imm) {
31 return ((uint64_t)Imm) < 16;
32}
33static bool Predicate_VectorIndex0(int64_t Imm) {
34 return ((uint64_t)Imm) == 0;
35}
36static bool Predicate_imm0_255(int64_t Imm) {
37
38 return ((uint32_t)Imm) < 256;
39
40}
41static bool Predicate_simm8_32b(int64_t Imm) {
42 return Imm >= -128 && Imm < 128;
43}
44static bool Predicate_simm8_64b(int64_t Imm) {
45 return Imm >= -128 && Imm < 128;
46}
47static bool Predicate_uimm8_32b(int64_t Imm) {
48 return Imm >= 0 && Imm < 256;
49}
50static bool Predicate_uimm8_64b(int64_t Imm) {
51 return Imm >= 0 && Imm < 256;
52}
53static bool Predicate_simm6_32b(int64_t Imm) {
54 return Imm >= -32 && Imm < 32;
55}
56
57
58// FastEmit functions for AArch64ISD::ENTRY_PSTATE_SM.
59
60Register fastEmit_AArch64ISD_ENTRY_PSTATE_SM_MVT_i64_(MVT RetVT) {
61 if (RetVT.SimpleTy != MVT::i64)
62 return Register();
63 return fastEmitInst_(MachineInstOpcode: AArch64::EntryPStateSM, RC: &AArch64::GPR64RegClass);
64}
65
66Register fastEmit_AArch64ISD_ENTRY_PSTATE_SM_(MVT VT, MVT RetVT) {
67 switch (VT.SimpleTy) {
68 case MVT::i64: return fastEmit_AArch64ISD_ENTRY_PSTATE_SM_MVT_i64_(RetVT);
69 default: return Register();
70 }
71}
72
73// FastEmit functions for AArch64ISD::THREAD_POINTER.
74
75Register fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(MVT RetVT) {
76 if (RetVT.SimpleTy != MVT::i64)
77 return Register();
78 return fastEmitInst_(MachineInstOpcode: AArch64::MOVbaseTLS, RC: &AArch64::GPR64RegClass);
79}
80
81Register fastEmit_AArch64ISD_THREAD_POINTER_(MVT VT, MVT RetVT) {
82 switch (VT.SimpleTy) {
83 case MVT::i64: return fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(RetVT);
84 default: return Register();
85 }
86}
87
88// Top-level FastEmit function.
89
90Register fastEmit_(MVT VT, MVT RetVT, unsigned Opcode) override {
91 switch (Opcode) {
92 case AArch64ISD::ENTRY_PSTATE_SM: return fastEmit_AArch64ISD_ENTRY_PSTATE_SM_(VT, RetVT);
93 case AArch64ISD::THREAD_POINTER: return fastEmit_AArch64ISD_THREAD_POINTER_(VT, RetVT);
94 default: return Register();
95 }
96}
97
98// FastEmit functions for AArch64ISD::COALESCER_BARRIER.
99
100Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_bf16_r(MVT RetVT, Register Op0) {
101 if (RetVT.SimpleTy != MVT::bf16)
102 return Register();
103 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR16, RC: &AArch64::FPR16RegClass, Op0);
104}
105
106Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f16_r(MVT RetVT, Register Op0) {
107 if (RetVT.SimpleTy != MVT::f16)
108 return Register();
109 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR16, RC: &AArch64::FPR16RegClass, Op0);
110}
111
112Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f32_r(MVT RetVT, Register Op0) {
113 if (RetVT.SimpleTy != MVT::f32)
114 return Register();
115 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR32, RC: &AArch64::FPR32RegClass, Op0);
116}
117
118Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f64_r(MVT RetVT, Register Op0) {
119 if (RetVT.SimpleTy != MVT::f64)
120 return Register();
121 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
122}
123
124Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f128_r(MVT RetVT, Register Op0) {
125 if (RetVT.SimpleTy != MVT::f128)
126 return Register();
127 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
128}
129
130Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i8_r(MVT RetVT, Register Op0) {
131 if (RetVT.SimpleTy != MVT::v8i8)
132 return Register();
133 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
134}
135
136Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v16i8_r(MVT RetVT, Register Op0) {
137 if (RetVT.SimpleTy != MVT::v16i8)
138 return Register();
139 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
140}
141
142Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i16_r(MVT RetVT, Register Op0) {
143 if (RetVT.SimpleTy != MVT::v4i16)
144 return Register();
145 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
146}
147
148Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i16_r(MVT RetVT, Register Op0) {
149 if (RetVT.SimpleTy != MVT::v8i16)
150 return Register();
151 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
152}
153
154Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i32_r(MVT RetVT, Register Op0) {
155 if (RetVT.SimpleTy != MVT::v2i32)
156 return Register();
157 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
158}
159
160Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i32_r(MVT RetVT, Register Op0) {
161 if (RetVT.SimpleTy != MVT::v4i32)
162 return Register();
163 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
164}
165
166Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1i64_r(MVT RetVT, Register Op0) {
167 if (RetVT.SimpleTy != MVT::v1i64)
168 return Register();
169 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
170}
171
172Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i64_r(MVT RetVT, Register Op0) {
173 if (RetVT.SimpleTy != MVT::v2i64)
174 return Register();
175 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
176}
177
178Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f16_r(MVT RetVT, Register Op0) {
179 if (RetVT.SimpleTy != MVT::v4f16)
180 return Register();
181 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
182}
183
184Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8f16_r(MVT RetVT, Register Op0) {
185 if (RetVT.SimpleTy != MVT::v8f16)
186 return Register();
187 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
188}
189
190Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4bf16_r(MVT RetVT, Register Op0) {
191 if (RetVT.SimpleTy != MVT::v4bf16)
192 return Register();
193 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
194}
195
196Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8bf16_r(MVT RetVT, Register Op0) {
197 if (RetVT.SimpleTy != MVT::v8bf16)
198 return Register();
199 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
200}
201
202Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f32_r(MVT RetVT, Register Op0) {
203 if (RetVT.SimpleTy != MVT::v2f32)
204 return Register();
205 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
206}
207
208Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f32_r(MVT RetVT, Register Op0) {
209 if (RetVT.SimpleTy != MVT::v4f32)
210 return Register();
211 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
212}
213
214Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1f64_r(MVT RetVT, Register Op0) {
215 if (RetVT.SimpleTy != MVT::v1f64)
216 return Register();
217 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
218}
219
220Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f64_r(MVT RetVT, Register Op0) {
221 if (RetVT.SimpleTy != MVT::v2f64)
222 return Register();
223 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
224}
225
226Register fastEmit_AArch64ISD_COALESCER_BARRIER_r(MVT VT, MVT RetVT, Register Op0) {
227 switch (VT.SimpleTy) {
228 case MVT::bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_bf16_r(RetVT, Op0);
229 case MVT::f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f16_r(RetVT, Op0);
230 case MVT::f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f32_r(RetVT, Op0);
231 case MVT::f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f64_r(RetVT, Op0);
232 case MVT::f128: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f128_r(RetVT, Op0);
233 case MVT::v8i8: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i8_r(RetVT, Op0);
234 case MVT::v16i8: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v16i8_r(RetVT, Op0);
235 case MVT::v4i16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i16_r(RetVT, Op0);
236 case MVT::v8i16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i16_r(RetVT, Op0);
237 case MVT::v2i32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i32_r(RetVT, Op0);
238 case MVT::v4i32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i32_r(RetVT, Op0);
239 case MVT::v1i64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1i64_r(RetVT, Op0);
240 case MVT::v2i64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i64_r(RetVT, Op0);
241 case MVT::v4f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f16_r(RetVT, Op0);
242 case MVT::v8f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8f16_r(RetVT, Op0);
243 case MVT::v4bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4bf16_r(RetVT, Op0);
244 case MVT::v8bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8bf16_r(RetVT, Op0);
245 case MVT::v2f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f32_r(RetVT, Op0);
246 case MVT::v4f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f32_r(RetVT, Op0);
247 case MVT::v1f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1f64_r(RetVT, Op0);
248 case MVT::v2f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f64_r(RetVT, Op0);
249 default: return Register();
250 }
251}
252
253// FastEmit functions for AArch64ISD::DUP.
254
255Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Register Op0) {
256 if ((Subtarget->isNeonAvailable())) {
257 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv8i8gpr, RC: &AArch64::FPR64RegClass, Op0);
258 }
259 return Register();
260}
261
262Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Register Op0) {
263 if ((Subtarget->isNeonAvailable())) {
264 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv16i8gpr, RC: &AArch64::FPR128RegClass, Op0);
265 }
266 return Register();
267}
268
269Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Register Op0) {
270 if ((Subtarget->isNeonAvailable())) {
271 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv4i16gpr, RC: &AArch64::FPR64RegClass, Op0);
272 }
273 return Register();
274}
275
276Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Register Op0) {
277 if ((Subtarget->isNeonAvailable())) {
278 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv8i16gpr, RC: &AArch64::FPR128RegClass, Op0);
279 }
280 return Register();
281}
282
283Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Register Op0) {
284 if ((Subtarget->isNeonAvailable())) {
285 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv2i32gpr, RC: &AArch64::FPR64RegClass, Op0);
286 }
287 return Register();
288}
289
290Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Register Op0) {
291 if ((Subtarget->isNeonAvailable())) {
292 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv4i32gpr, RC: &AArch64::FPR128RegClass, Op0);
293 }
294 return Register();
295}
296
297Register fastEmit_AArch64ISD_DUP_MVT_i32_r(MVT RetVT, Register Op0) {
298switch (RetVT.SimpleTy) {
299 case MVT::v8i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Op0);
300 case MVT::v16i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Op0);
301 case MVT::v4i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Op0);
302 case MVT::v8i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Op0);
303 case MVT::v2i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Op0);
304 case MVT::v4i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Op0);
305 default: return Register();
306}
307}
308
309Register fastEmit_AArch64ISD_DUP_MVT_i64_r(MVT RetVT, Register Op0) {
310 if (RetVT.SimpleTy != MVT::v2i64)
311 return Register();
312 if ((Subtarget->isNeonAvailable())) {
313 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv2i64gpr, RC: &AArch64::FPR128RegClass, Op0);
314 }
315 return Register();
316}
317
318Register fastEmit_AArch64ISD_DUP_r(MVT VT, MVT RetVT, Register Op0) {
319 switch (VT.SimpleTy) {
320 case MVT::i32: return fastEmit_AArch64ISD_DUP_MVT_i32_r(RetVT, Op0);
321 case MVT::i64: return fastEmit_AArch64ISD_DUP_MVT_i64_r(RetVT, Op0);
322 default: return Register();
323 }
324}
325
326// FastEmit functions for AArch64ISD::FCVTXN.
327
328Register fastEmit_AArch64ISD_FCVTXN_MVT_f64_r(MVT RetVT, Register Op0) {
329 if (RetVT.SimpleTy != MVT::f32)
330 return Register();
331 if ((Subtarget->isNeonAvailable())) {
332 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTXNv1i64, RC: &AArch64::FPR32RegClass, Op0);
333 }
334 return Register();
335}
336
337Register fastEmit_AArch64ISD_FCVTXN_MVT_v2f64_r(MVT RetVT, Register Op0) {
338 if (RetVT.SimpleTy != MVT::v2f32)
339 return Register();
340 if ((Subtarget->isNeonAvailable())) {
341 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTXNv2f32, RC: &AArch64::FPR64RegClass, Op0);
342 }
343 return Register();
344}
345
346Register fastEmit_AArch64ISD_FCVTXN_r(MVT VT, MVT RetVT, Register Op0) {
347 switch (VT.SimpleTy) {
348 case MVT::f64: return fastEmit_AArch64ISD_FCVTXN_MVT_f64_r(RetVT, Op0);
349 case MVT::v2f64: return fastEmit_AArch64ISD_FCVTXN_MVT_v2f64_r(RetVT, Op0);
350 default: return Register();
351 }
352}
353
354// FastEmit functions for AArch64ISD::FRECPE.
355
356Register fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(MVT RetVT, Register Op0) {
357 if (RetVT.SimpleTy != MVT::v2f32)
358 return Register();
359 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv2f32, RC: &AArch64::FPR64RegClass, Op0);
360}
361
362Register fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(MVT RetVT, Register Op0) {
363 if (RetVT.SimpleTy != MVT::v4f32)
364 return Register();
365 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv4f32, RC: &AArch64::FPR128RegClass, Op0);
366}
367
368Register fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(MVT RetVT, Register Op0) {
369 if (RetVT.SimpleTy != MVT::v2f64)
370 return Register();
371 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv2f64, RC: &AArch64::FPR128RegClass, Op0);
372}
373
374Register fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
375 if (RetVT.SimpleTy != MVT::nxv8f16)
376 return Register();
377 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
378 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
379 }
380 return Register();
381}
382
383Register fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
384 if (RetVT.SimpleTy != MVT::nxv4f32)
385 return Register();
386 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
387 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
388 }
389 return Register();
390}
391
392Register fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
393 if (RetVT.SimpleTy != MVT::nxv2f64)
394 return Register();
395 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
396 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
397 }
398 return Register();
399}
400
401Register fastEmit_AArch64ISD_FRECPE_r(MVT VT, MVT RetVT, Register Op0) {
402 switch (VT.SimpleTy) {
403 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(RetVT, Op0);
404 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(RetVT, Op0);
405 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(RetVT, Op0);
406 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(RetVT, Op0);
407 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(RetVT, Op0);
408 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(RetVT, Op0);
409 default: return Register();
410 }
411}
412
413// FastEmit functions for AArch64ISD::FRSQRTE.
414
415Register fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(MVT RetVT, Register Op0) {
416 if (RetVT.SimpleTy != MVT::v2f32)
417 return Register();
418 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv2f32, RC: &AArch64::FPR64RegClass, Op0);
419}
420
421Register fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(MVT RetVT, Register Op0) {
422 if (RetVT.SimpleTy != MVT::v4f32)
423 return Register();
424 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv4f32, RC: &AArch64::FPR128RegClass, Op0);
425}
426
427Register fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(MVT RetVT, Register Op0) {
428 if (RetVT.SimpleTy != MVT::v2f64)
429 return Register();
430 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv2f64, RC: &AArch64::FPR128RegClass, Op0);
431}
432
433Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
434 if (RetVT.SimpleTy != MVT::nxv8f16)
435 return Register();
436 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
437 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
438 }
439 return Register();
440}
441
442Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
443 if (RetVT.SimpleTy != MVT::nxv4f32)
444 return Register();
445 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
446 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
447 }
448 return Register();
449}
450
451Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
452 if (RetVT.SimpleTy != MVT::nxv2f64)
453 return Register();
454 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
455 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
456 }
457 return Register();
458}
459
460Register fastEmit_AArch64ISD_FRSQRTE_r(MVT VT, MVT RetVT, Register Op0) {
461 switch (VT.SimpleTy) {
462 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(RetVT, Op0);
463 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(RetVT, Op0);
464 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(RetVT, Op0);
465 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(RetVT, Op0);
466 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(RetVT, Op0);
467 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(RetVT, Op0);
468 default: return Register();
469 }
470}
471
472// FastEmit functions for AArch64ISD::REV16.
473
474Register fastEmit_AArch64ISD_REV16_MVT_i32_r(MVT RetVT, Register Op0) {
475 if (RetVT.SimpleTy != MVT::i32)
476 return Register();
477 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16Wr, RC: &AArch64::GPR32RegClass, Op0);
478}
479
480Register fastEmit_AArch64ISD_REV16_MVT_i64_r(MVT RetVT, Register Op0) {
481 if (RetVT.SimpleTy != MVT::i64)
482 return Register();
483 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16Xr, RC: &AArch64::GPR64RegClass, Op0);
484}
485
486Register fastEmit_AArch64ISD_REV16_r(MVT VT, MVT RetVT, Register Op0) {
487 switch (VT.SimpleTy) {
488 case MVT::i32: return fastEmit_AArch64ISD_REV16_MVT_i32_r(RetVT, Op0);
489 case MVT::i64: return fastEmit_AArch64ISD_REV16_MVT_i64_r(RetVT, Op0);
490 default: return Register();
491 }
492}
493
494// FastEmit functions for AArch64ISD::REV32.
495
496Register fastEmit_AArch64ISD_REV32_MVT_v8i8_r(MVT RetVT, Register Op0) {
497 if (RetVT.SimpleTy != MVT::v8i8)
498 return Register();
499 if ((Subtarget->isNeonAvailable())) {
500 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
501 }
502 return Register();
503}
504
505Register fastEmit_AArch64ISD_REV32_MVT_v16i8_r(MVT RetVT, Register Op0) {
506 if (RetVT.SimpleTy != MVT::v16i8)
507 return Register();
508 if ((Subtarget->isNeonAvailable())) {
509 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
510 }
511 return Register();
512}
513
514Register fastEmit_AArch64ISD_REV32_MVT_v4i16_r(MVT RetVT, Register Op0) {
515 if (RetVT.SimpleTy != MVT::v4i16)
516 return Register();
517 if ((Subtarget->isNeonAvailable())) {
518 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
519 }
520 return Register();
521}
522
523Register fastEmit_AArch64ISD_REV32_MVT_v8i16_r(MVT RetVT, Register Op0) {
524 if (RetVT.SimpleTy != MVT::v8i16)
525 return Register();
526 if ((Subtarget->isNeonAvailable())) {
527 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
528 }
529 return Register();
530}
531
532Register fastEmit_AArch64ISD_REV32_MVT_v4f16_r(MVT RetVT, Register Op0) {
533 if (RetVT.SimpleTy != MVT::v4f16)
534 return Register();
535 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
536}
537
538Register fastEmit_AArch64ISD_REV32_MVT_v8f16_r(MVT RetVT, Register Op0) {
539 if (RetVT.SimpleTy != MVT::v8f16)
540 return Register();
541 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
542}
543
544Register fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(MVT RetVT, Register Op0) {
545 if (RetVT.SimpleTy != MVT::v4bf16)
546 return Register();
547 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
548}
549
550Register fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(MVT RetVT, Register Op0) {
551 if (RetVT.SimpleTy != MVT::v8bf16)
552 return Register();
553 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
554}
555
556Register fastEmit_AArch64ISD_REV32_r(MVT VT, MVT RetVT, Register Op0) {
557 switch (VT.SimpleTy) {
558 case MVT::v8i8: return fastEmit_AArch64ISD_REV32_MVT_v8i8_r(RetVT, Op0);
559 case MVT::v16i8: return fastEmit_AArch64ISD_REV32_MVT_v16i8_r(RetVT, Op0);
560 case MVT::v4i16: return fastEmit_AArch64ISD_REV32_MVT_v4i16_r(RetVT, Op0);
561 case MVT::v8i16: return fastEmit_AArch64ISD_REV32_MVT_v8i16_r(RetVT, Op0);
562 case MVT::v4f16: return fastEmit_AArch64ISD_REV32_MVT_v4f16_r(RetVT, Op0);
563 case MVT::v8f16: return fastEmit_AArch64ISD_REV32_MVT_v8f16_r(RetVT, Op0);
564 case MVT::v4bf16: return fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(RetVT, Op0);
565 case MVT::v8bf16: return fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(RetVT, Op0);
566 default: return Register();
567 }
568}
569
570// FastEmit functions for AArch64ISD::REV64.
571
572Register fastEmit_AArch64ISD_REV64_MVT_v8i8_r(MVT RetVT, Register Op0) {
573 if (RetVT.SimpleTy != MVT::v8i8)
574 return Register();
575 if ((Subtarget->isNeonAvailable())) {
576 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
577 }
578 return Register();
579}
580
581Register fastEmit_AArch64ISD_REV64_MVT_v16i8_r(MVT RetVT, Register Op0) {
582 if (RetVT.SimpleTy != MVT::v16i8)
583 return Register();
584 if ((Subtarget->isNeonAvailable())) {
585 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
586 }
587 return Register();
588}
589
590Register fastEmit_AArch64ISD_REV64_MVT_v4i16_r(MVT RetVT, Register Op0) {
591 if (RetVT.SimpleTy != MVT::v4i16)
592 return Register();
593 if ((Subtarget->isNeonAvailable())) {
594 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
595 }
596 return Register();
597}
598
599Register fastEmit_AArch64ISD_REV64_MVT_v8i16_r(MVT RetVT, Register Op0) {
600 if (RetVT.SimpleTy != MVT::v8i16)
601 return Register();
602 if ((Subtarget->isNeonAvailable())) {
603 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
604 }
605 return Register();
606}
607
608Register fastEmit_AArch64ISD_REV64_MVT_v2i32_r(MVT RetVT, Register Op0) {
609 if (RetVT.SimpleTy != MVT::v2i32)
610 return Register();
611 if ((Subtarget->isNeonAvailable())) {
612 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
613 }
614 return Register();
615}
616
617Register fastEmit_AArch64ISD_REV64_MVT_v4i32_r(MVT RetVT, Register Op0) {
618 if (RetVT.SimpleTy != MVT::v4i32)
619 return Register();
620 if ((Subtarget->isNeonAvailable())) {
621 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
622 }
623 return Register();
624}
625
626Register fastEmit_AArch64ISD_REV64_MVT_v4f16_r(MVT RetVT, Register Op0) {
627 if (RetVT.SimpleTy != MVT::v4f16)
628 return Register();
629 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
630}
631
632Register fastEmit_AArch64ISD_REV64_MVT_v8f16_r(MVT RetVT, Register Op0) {
633 if (RetVT.SimpleTy != MVT::v8f16)
634 return Register();
635 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
636}
637
638Register fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(MVT RetVT, Register Op0) {
639 if (RetVT.SimpleTy != MVT::v4bf16)
640 return Register();
641 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
642}
643
644Register fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(MVT RetVT, Register Op0) {
645 if (RetVT.SimpleTy != MVT::v8bf16)
646 return Register();
647 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
648}
649
650Register fastEmit_AArch64ISD_REV64_MVT_v2f32_r(MVT RetVT, Register Op0) {
651 if (RetVT.SimpleTy != MVT::v2f32)
652 return Register();
653 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
654}
655
656Register fastEmit_AArch64ISD_REV64_MVT_v4f32_r(MVT RetVT, Register Op0) {
657 if (RetVT.SimpleTy != MVT::v4f32)
658 return Register();
659 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
660}
661
662Register fastEmit_AArch64ISD_REV64_r(MVT VT, MVT RetVT, Register Op0) {
663 switch (VT.SimpleTy) {
664 case MVT::v8i8: return fastEmit_AArch64ISD_REV64_MVT_v8i8_r(RetVT, Op0);
665 case MVT::v16i8: return fastEmit_AArch64ISD_REV64_MVT_v16i8_r(RetVT, Op0);
666 case MVT::v4i16: return fastEmit_AArch64ISD_REV64_MVT_v4i16_r(RetVT, Op0);
667 case MVT::v8i16: return fastEmit_AArch64ISD_REV64_MVT_v8i16_r(RetVT, Op0);
668 case MVT::v2i32: return fastEmit_AArch64ISD_REV64_MVT_v2i32_r(RetVT, Op0);
669 case MVT::v4i32: return fastEmit_AArch64ISD_REV64_MVT_v4i32_r(RetVT, Op0);
670 case MVT::v4f16: return fastEmit_AArch64ISD_REV64_MVT_v4f16_r(RetVT, Op0);
671 case MVT::v8f16: return fastEmit_AArch64ISD_REV64_MVT_v8f16_r(RetVT, Op0);
672 case MVT::v4bf16: return fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(RetVT, Op0);
673 case MVT::v8bf16: return fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(RetVT, Op0);
674 case MVT::v2f32: return fastEmit_AArch64ISD_REV64_MVT_v2f32_r(RetVT, Op0);
675 case MVT::v4f32: return fastEmit_AArch64ISD_REV64_MVT_v4f32_r(RetVT, Op0);
676 default: return Register();
677 }
678}
679
680// FastEmit functions for AArch64ISD::SADDLP.
681
682Register fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(MVT RetVT, Register Op0) {
683 if (RetVT.SimpleTy != MVT::v4i16)
684 return Register();
685 if ((Subtarget->isNeonAvailable())) {
686 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv8i8_v4i16, RC: &AArch64::FPR64RegClass, Op0);
687 }
688 return Register();
689}
690
691Register fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(MVT RetVT, Register Op0) {
692 if (RetVT.SimpleTy != MVT::v8i16)
693 return Register();
694 if ((Subtarget->isNeonAvailable())) {
695 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv16i8_v8i16, RC: &AArch64::FPR128RegClass, Op0);
696 }
697 return Register();
698}
699
700Register fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(MVT RetVT, Register Op0) {
701 if (RetVT.SimpleTy != MVT::v2i32)
702 return Register();
703 if ((Subtarget->isNeonAvailable())) {
704 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv4i16_v2i32, RC: &AArch64::FPR64RegClass, Op0);
705 }
706 return Register();
707}
708
709Register fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(MVT RetVT, Register Op0) {
710 if (RetVT.SimpleTy != MVT::v4i32)
711 return Register();
712 if ((Subtarget->isNeonAvailable())) {
713 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv8i16_v4i32, RC: &AArch64::FPR128RegClass, Op0);
714 }
715 return Register();
716}
717
718Register fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(MVT RetVT, Register Op0) {
719 if (RetVT.SimpleTy != MVT::v1i64)
720 return Register();
721 if ((Subtarget->isNeonAvailable())) {
722 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv2i32_v1i64, RC: &AArch64::FPR64RegClass, Op0);
723 }
724 return Register();
725}
726
727Register fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(MVT RetVT, Register Op0) {
728 if (RetVT.SimpleTy != MVT::v2i64)
729 return Register();
730 if ((Subtarget->isNeonAvailable())) {
731 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv4i32_v2i64, RC: &AArch64::FPR128RegClass, Op0);
732 }
733 return Register();
734}
735
736Register fastEmit_AArch64ISD_SADDLP_r(MVT VT, MVT RetVT, Register Op0) {
737 switch (VT.SimpleTy) {
738 case MVT::v8i8: return fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(RetVT, Op0);
739 case MVT::v16i8: return fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(RetVT, Op0);
740 case MVT::v4i16: return fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(RetVT, Op0);
741 case MVT::v8i16: return fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(RetVT, Op0);
742 case MVT::v2i32: return fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(RetVT, Op0);
743 case MVT::v4i32: return fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(RetVT, Op0);
744 default: return Register();
745 }
746}
747
748// FastEmit functions for AArch64ISD::SITOF.
749
750Register fastEmit_AArch64ISD_SITOF_MVT_f16_r(MVT RetVT, Register Op0) {
751 if (RetVT.SimpleTy != MVT::f16)
752 return Register();
753 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
754 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i16, RC: &AArch64::FPR16RegClass, Op0);
755 }
756 return Register();
757}
758
759Register fastEmit_AArch64ISD_SITOF_MVT_f32_r(MVT RetVT, Register Op0) {
760 if (RetVT.SimpleTy != MVT::f32)
761 return Register();
762 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
763 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i32, RC: &AArch64::FPR32RegClass, Op0);
764 }
765 return Register();
766}
767
768Register fastEmit_AArch64ISD_SITOF_MVT_f64_r(MVT RetVT, Register Op0) {
769 if (RetVT.SimpleTy != MVT::f64)
770 return Register();
771 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
772 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i64, RC: &AArch64::FPR64RegClass, Op0);
773 }
774 return Register();
775}
776
777Register fastEmit_AArch64ISD_SITOF_r(MVT VT, MVT RetVT, Register Op0) {
778 switch (VT.SimpleTy) {
779 case MVT::f16: return fastEmit_AArch64ISD_SITOF_MVT_f16_r(RetVT, Op0);
780 case MVT::f32: return fastEmit_AArch64ISD_SITOF_MVT_f32_r(RetVT, Op0);
781 case MVT::f64: return fastEmit_AArch64ISD_SITOF_MVT_f64_r(RetVT, Op0);
782 default: return Register();
783 }
784}
785
786// FastEmit functions for AArch64ISD::SQABS.
787
788Register fastEmit_AArch64ISD_SQABS_MVT_f32_r(MVT RetVT, Register Op0) {
789 if (RetVT.SimpleTy != MVT::f32)
790 return Register();
791 if ((Subtarget->isNeonAvailable())) {
792 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv1i32, RC: &AArch64::FPR32RegClass, Op0);
793 }
794 return Register();
795}
796
797Register fastEmit_AArch64ISD_SQABS_MVT_f64_r(MVT RetVT, Register Op0) {
798 if (RetVT.SimpleTy != MVT::f64)
799 return Register();
800 if ((Subtarget->isNeonAvailable())) {
801 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
802 }
803 return Register();
804}
805
806Register fastEmit_AArch64ISD_SQABS_r(MVT VT, MVT RetVT, Register Op0) {
807 switch (VT.SimpleTy) {
808 case MVT::f32: return fastEmit_AArch64ISD_SQABS_MVT_f32_r(RetVT, Op0);
809 case MVT::f64: return fastEmit_AArch64ISD_SQABS_MVT_f64_r(RetVT, Op0);
810 default: return Register();
811 }
812}
813
814// FastEmit functions for AArch64ISD::SQNEG.
815
816Register fastEmit_AArch64ISD_SQNEG_MVT_f32_r(MVT RetVT, Register Op0) {
817 if (RetVT.SimpleTy != MVT::f32)
818 return Register();
819 if ((Subtarget->isNeonAvailable())) {
820 return fastEmitInst_r(MachineInstOpcode: AArch64::SQNEGv1i32, RC: &AArch64::FPR32RegClass, Op0);
821 }
822 return Register();
823}
824
825Register fastEmit_AArch64ISD_SQNEG_MVT_f64_r(MVT RetVT, Register Op0) {
826 if (RetVT.SimpleTy != MVT::f64)
827 return Register();
828 if ((Subtarget->isNeonAvailable())) {
829 return fastEmitInst_r(MachineInstOpcode: AArch64::SQNEGv1i64, RC: &AArch64::FPR64RegClass, Op0);
830 }
831 return Register();
832}
833
834Register fastEmit_AArch64ISD_SQNEG_r(MVT VT, MVT RetVT, Register Op0) {
835 switch (VT.SimpleTy) {
836 case MVT::f32: return fastEmit_AArch64ISD_SQNEG_MVT_f32_r(RetVT, Op0);
837 case MVT::f64: return fastEmit_AArch64ISD_SQNEG_MVT_f64_r(RetVT, Op0);
838 default: return Register();
839 }
840}
841
842// FastEmit functions for AArch64ISD::SUNPKHI.
843
844Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
845 if (RetVT.SimpleTy != MVT::nxv8i16)
846 return Register();
847 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
848 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
849 }
850 return Register();
851}
852
853Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
854 if (RetVT.SimpleTy != MVT::nxv4i32)
855 return Register();
856 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
857 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
858 }
859 return Register();
860}
861
862Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
863 if (RetVT.SimpleTy != MVT::nxv2i64)
864 return Register();
865 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
866 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
867 }
868 return Register();
869}
870
871Register fastEmit_AArch64ISD_SUNPKHI_r(MVT VT, MVT RetVT, Register Op0) {
872 switch (VT.SimpleTy) {
873 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(RetVT, Op0);
874 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(RetVT, Op0);
875 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(RetVT, Op0);
876 default: return Register();
877 }
878}
879
880// FastEmit functions for AArch64ISD::SUNPKLO.
881
882Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
883 if (RetVT.SimpleTy != MVT::nxv8i16)
884 return Register();
885 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
886 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
887 }
888 return Register();
889}
890
891Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
892 if (RetVT.SimpleTy != MVT::nxv4i32)
893 return Register();
894 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
895 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
896 }
897 return Register();
898}
899
900Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
901 if (RetVT.SimpleTy != MVT::nxv2i64)
902 return Register();
903 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
904 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
905 }
906 return Register();
907}
908
909Register fastEmit_AArch64ISD_SUNPKLO_r(MVT VT, MVT RetVT, Register Op0) {
910 switch (VT.SimpleTy) {
911 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(RetVT, Op0);
912 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(RetVT, Op0);
913 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(RetVT, Op0);
914 default: return Register();
915 }
916}
917
918// FastEmit functions for AArch64ISD::UADDLP.
919
920Register fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(MVT RetVT, Register Op0) {
921 if (RetVT.SimpleTy != MVT::v4i16)
922 return Register();
923 if ((Subtarget->isNeonAvailable())) {
924 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv8i8_v4i16, RC: &AArch64::FPR64RegClass, Op0);
925 }
926 return Register();
927}
928
929Register fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(MVT RetVT, Register Op0) {
930 if (RetVT.SimpleTy != MVT::v8i16)
931 return Register();
932 if ((Subtarget->isNeonAvailable())) {
933 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv16i8_v8i16, RC: &AArch64::FPR128RegClass, Op0);
934 }
935 return Register();
936}
937
938Register fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(MVT RetVT, Register Op0) {
939 if (RetVT.SimpleTy != MVT::v2i32)
940 return Register();
941 if ((Subtarget->isNeonAvailable())) {
942 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv4i16_v2i32, RC: &AArch64::FPR64RegClass, Op0);
943 }
944 return Register();
945}
946
947Register fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(MVT RetVT, Register Op0) {
948 if (RetVT.SimpleTy != MVT::v4i32)
949 return Register();
950 if ((Subtarget->isNeonAvailable())) {
951 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv8i16_v4i32, RC: &AArch64::FPR128RegClass, Op0);
952 }
953 return Register();
954}
955
956Register fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(MVT RetVT, Register Op0) {
957 if (RetVT.SimpleTy != MVT::v1i64)
958 return Register();
959 if ((Subtarget->isNeonAvailable())) {
960 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv2i32_v1i64, RC: &AArch64::FPR64RegClass, Op0);
961 }
962 return Register();
963}
964
965Register fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(MVT RetVT, Register Op0) {
966 if (RetVT.SimpleTy != MVT::v2i64)
967 return Register();
968 if ((Subtarget->isNeonAvailable())) {
969 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv4i32_v2i64, RC: &AArch64::FPR128RegClass, Op0);
970 }
971 return Register();
972}
973
974Register fastEmit_AArch64ISD_UADDLP_r(MVT VT, MVT RetVT, Register Op0) {
975 switch (VT.SimpleTy) {
976 case MVT::v8i8: return fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(RetVT, Op0);
977 case MVT::v16i8: return fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(RetVT, Op0);
978 case MVT::v4i16: return fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(RetVT, Op0);
979 case MVT::v8i16: return fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(RetVT, Op0);
980 case MVT::v2i32: return fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(RetVT, Op0);
981 case MVT::v4i32: return fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(RetVT, Op0);
982 default: return Register();
983 }
984}
985
986// FastEmit functions for AArch64ISD::UITOF.
987
988Register fastEmit_AArch64ISD_UITOF_MVT_f16_r(MVT RetVT, Register Op0) {
989 if (RetVT.SimpleTy != MVT::f16)
990 return Register();
991 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
992 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i16, RC: &AArch64::FPR16RegClass, Op0);
993 }
994 return Register();
995}
996
997Register fastEmit_AArch64ISD_UITOF_MVT_f32_r(MVT RetVT, Register Op0) {
998 if (RetVT.SimpleTy != MVT::f32)
999 return Register();
1000 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
1001 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i32, RC: &AArch64::FPR32RegClass, Op0);
1002 }
1003 return Register();
1004}
1005
1006Register fastEmit_AArch64ISD_UITOF_MVT_f64_r(MVT RetVT, Register Op0) {
1007 if (RetVT.SimpleTy != MVT::f64)
1008 return Register();
1009 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
1010 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i64, RC: &AArch64::FPR64RegClass, Op0);
1011 }
1012 return Register();
1013}
1014
1015Register fastEmit_AArch64ISD_UITOF_r(MVT VT, MVT RetVT, Register Op0) {
1016 switch (VT.SimpleTy) {
1017 case MVT::f16: return fastEmit_AArch64ISD_UITOF_MVT_f16_r(RetVT, Op0);
1018 case MVT::f32: return fastEmit_AArch64ISD_UITOF_MVT_f32_r(RetVT, Op0);
1019 case MVT::f64: return fastEmit_AArch64ISD_UITOF_MVT_f64_r(RetVT, Op0);
1020 default: return Register();
1021 }
1022}
1023
1024// FastEmit functions for AArch64ISD::UUNPKHI.
1025
1026Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
1027 if (RetVT.SimpleTy != MVT::nxv8i16)
1028 return Register();
1029 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1030 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1031 }
1032 return Register();
1033}
1034
1035Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
1036 if (RetVT.SimpleTy != MVT::nxv4i32)
1037 return Register();
1038 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1039 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1040 }
1041 return Register();
1042}
1043
1044Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
1045 if (RetVT.SimpleTy != MVT::nxv2i64)
1046 return Register();
1047 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1048 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1049 }
1050 return Register();
1051}
1052
1053Register fastEmit_AArch64ISD_UUNPKHI_r(MVT VT, MVT RetVT, Register Op0) {
1054 switch (VT.SimpleTy) {
1055 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(RetVT, Op0);
1056 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(RetVT, Op0);
1057 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(RetVT, Op0);
1058 default: return Register();
1059 }
1060}
1061
1062// FastEmit functions for AArch64ISD::UUNPKLO.
1063
1064Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
1065 if (RetVT.SimpleTy != MVT::nxv8i16)
1066 return Register();
1067 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1068 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1069 }
1070 return Register();
1071}
1072
1073Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
1074 if (RetVT.SimpleTy != MVT::nxv4i32)
1075 return Register();
1076 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1077 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1078 }
1079 return Register();
1080}
1081
1082Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
1083 if (RetVT.SimpleTy != MVT::nxv2i64)
1084 return Register();
1085 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1086 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1087 }
1088 return Register();
1089}
1090
1091Register fastEmit_AArch64ISD_UUNPKLO_r(MVT VT, MVT RetVT, Register Op0) {
1092 switch (VT.SimpleTy) {
1093 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(RetVT, Op0);
1094 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(RetVT, Op0);
1095 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(RetVT, Op0);
1096 default: return Register();
1097 }
1098}
1099
1100// FastEmit functions for ISD::ABS.
1101
1102Register fastEmit_ISD_ABS_MVT_i32_r(MVT RetVT, Register Op0) {
1103 if (RetVT.SimpleTy != MVT::i32)
1104 return Register();
1105 if ((Subtarget->hasCSSC())) {
1106 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSWr, RC: &AArch64::GPR32RegClass, Op0);
1107 }
1108 return Register();
1109}
1110
1111Register fastEmit_ISD_ABS_MVT_i64_r(MVT RetVT, Register Op0) {
1112 if (RetVT.SimpleTy != MVT::i64)
1113 return Register();
1114 if ((!Subtarget->hasCSSC())) {
1115 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
1116 }
1117 if ((Subtarget->hasCSSC())) {
1118 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSXr, RC: &AArch64::GPR64RegClass, Op0);
1119 }
1120 return Register();
1121}
1122
1123Register fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, Register Op0) {
1124 if (RetVT.SimpleTy != MVT::v8i8)
1125 return Register();
1126 if ((Subtarget->isNeonAvailable())) {
1127 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv8i8, RC: &AArch64::FPR64RegClass, Op0);
1128 }
1129 return Register();
1130}
1131
1132Register fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, Register Op0) {
1133 if (RetVT.SimpleTy != MVT::v16i8)
1134 return Register();
1135 if ((Subtarget->isNeonAvailable())) {
1136 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv16i8, RC: &AArch64::FPR128RegClass, Op0);
1137 }
1138 return Register();
1139}
1140
1141Register fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, Register Op0) {
1142 if (RetVT.SimpleTy != MVT::v4i16)
1143 return Register();
1144 if ((Subtarget->isNeonAvailable())) {
1145 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv4i16, RC: &AArch64::FPR64RegClass, Op0);
1146 }
1147 return Register();
1148}
1149
1150Register fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, Register Op0) {
1151 if (RetVT.SimpleTy != MVT::v8i16)
1152 return Register();
1153 if ((Subtarget->isNeonAvailable())) {
1154 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv8i16, RC: &AArch64::FPR128RegClass, Op0);
1155 }
1156 return Register();
1157}
1158
1159Register fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, Register Op0) {
1160 if (RetVT.SimpleTy != MVT::v2i32)
1161 return Register();
1162 if ((Subtarget->isNeonAvailable())) {
1163 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv2i32, RC: &AArch64::FPR64RegClass, Op0);
1164 }
1165 return Register();
1166}
1167
1168Register fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, Register Op0) {
1169 if (RetVT.SimpleTy != MVT::v4i32)
1170 return Register();
1171 if ((Subtarget->isNeonAvailable())) {
1172 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv4i32, RC: &AArch64::FPR128RegClass, Op0);
1173 }
1174 return Register();
1175}
1176
1177Register fastEmit_ISD_ABS_MVT_v1i64_r(MVT RetVT, Register Op0) {
1178 if (RetVT.SimpleTy != MVT::v1i64)
1179 return Register();
1180 if ((Subtarget->isNeonAvailable())) {
1181 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
1182 }
1183 return Register();
1184}
1185
1186Register fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, Register Op0) {
1187 if (RetVT.SimpleTy != MVT::v2i64)
1188 return Register();
1189 if ((Subtarget->isNeonAvailable())) {
1190 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv2i64, RC: &AArch64::FPR128RegClass, Op0);
1191 }
1192 return Register();
1193}
1194
1195Register fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, Register Op0) {
1196 switch (VT.SimpleTy) {
1197 case MVT::i32: return fastEmit_ISD_ABS_MVT_i32_r(RetVT, Op0);
1198 case MVT::i64: return fastEmit_ISD_ABS_MVT_i64_r(RetVT, Op0);
1199 case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0);
1200 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
1201 case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0);
1202 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
1203 case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0);
1204 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
1205 case MVT::v1i64: return fastEmit_ISD_ABS_MVT_v1i64_r(RetVT, Op0);
1206 case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0);
1207 default: return Register();
1208 }
1209}
1210
1211// FastEmit functions for ISD::BITCAST.
1212
1213Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Register Op0) {
1214 if ((!Subtarget->isLittleEndian())) {
1215 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1216 }
1217 return Register();
1218}
1219
1220Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Register Op0) {
1221 if ((!Subtarget->isLittleEndian())) {
1222 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1223 }
1224 return Register();
1225}
1226
1227Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Register Op0) {
1228 if ((!Subtarget->isLittleEndian())) {
1229 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1230 }
1231 return Register();
1232}
1233
1234Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Register Op0) {
1235 if ((!Subtarget->isLittleEndian())) {
1236 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1237 }
1238 return Register();
1239}
1240
1241Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Register Op0) {
1242 if ((!Subtarget->isLittleEndian())) {
1243 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1244 }
1245 return Register();
1246}
1247
1248Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Register Op0) {
1249 if ((!Subtarget->isLittleEndian())) {
1250 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1251 }
1252 return Register();
1253}
1254
1255Register fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, Register Op0) {
1256switch (RetVT.SimpleTy) {
1257 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0);
1258 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0);
1259 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0);
1260 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0);
1261 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Op0);
1262 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0);
1263 default: return Register();
1264}
1265}
1266
1267Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Register Op0) {
1268 if ((!Subtarget->isLittleEndian())) {
1269 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1270 }
1271 return Register();
1272}
1273
1274Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Register Op0) {
1275 if ((!Subtarget->isLittleEndian())) {
1276 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1277 }
1278 return Register();
1279}
1280
1281Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Register Op0) {
1282 if ((!Subtarget->isLittleEndian())) {
1283 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1284 }
1285 return Register();
1286}
1287
1288Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Register Op0) {
1289 if ((!Subtarget->isLittleEndian())) {
1290 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1291 }
1292 return Register();
1293}
1294
1295Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Register Op0) {
1296 if ((!Subtarget->isLittleEndian())) {
1297 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1298 }
1299 return Register();
1300}
1301
1302Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Register Op0) {
1303 if ((!Subtarget->isLittleEndian())) {
1304 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1305 }
1306 return Register();
1307}
1308
1309Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Register Op0) {
1310 if ((!Subtarget->isLittleEndian())) {
1311 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1312 }
1313 return Register();
1314}
1315
1316Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Register Op0) {
1317 if ((!Subtarget->isLittleEndian())) {
1318 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1319 }
1320 return Register();
1321}
1322
1323Register fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, Register Op0) {
1324switch (RetVT.SimpleTy) {
1325 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0);
1326 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0);
1327 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0);
1328 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0);
1329 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0);
1330 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Op0);
1331 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0);
1332 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Op0);
1333 default: return Register();
1334}
1335}
1336
1337Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Register Op0) {
1338 if ((!Subtarget->isLittleEndian())) {
1339 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1340 }
1341 return Register();
1342}
1343
1344Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Register Op0) {
1345 if ((!Subtarget->isLittleEndian())) {
1346 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1347 }
1348 return Register();
1349}
1350
1351Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Register Op0) {
1352 if ((!Subtarget->isLittleEndian())) {
1353 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1354 }
1355 return Register();
1356}
1357
1358Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Register Op0) {
1359 if ((!Subtarget->isLittleEndian())) {
1360 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1361 }
1362 return Register();
1363}
1364
1365Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Register Op0) {
1366 if ((!Subtarget->isLittleEndian())) {
1367 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1368 }
1369 return Register();
1370}
1371
1372Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Register Op0) {
1373 if ((!Subtarget->isLittleEndian())) {
1374 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1375 }
1376 return Register();
1377}
1378
1379Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Register Op0) {
1380 if ((!Subtarget->isLittleEndian())) {
1381 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1382 }
1383 return Register();
1384}
1385
1386Register fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, Register Op0) {
1387switch (RetVT.SimpleTy) {
1388 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0);
1389 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0);
1390 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0);
1391 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0);
1392 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Op0);
1393 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0);
1394 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0);
1395 default: return Register();
1396}
1397}
1398
1399Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Register Op0) {
1400 if ((!Subtarget->isLittleEndian())) {
1401 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1402 }
1403 return Register();
1404}
1405
1406Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Register Op0) {
1407 if ((!Subtarget->isLittleEndian())) {
1408 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1409 }
1410 return Register();
1411}
1412
1413Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Register Op0) {
1414 if ((!Subtarget->isLittleEndian())) {
1415 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1416 }
1417 return Register();
1418}
1419
1420Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Register Op0) {
1421 if ((!Subtarget->isLittleEndian())) {
1422 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1423 }
1424 return Register();
1425}
1426
1427Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Register Op0) {
1428 if ((!Subtarget->isLittleEndian())) {
1429 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1430 }
1431 return Register();
1432}
1433
1434Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Register Op0) {
1435 if ((!Subtarget->isLittleEndian())) {
1436 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1437 }
1438 return Register();
1439}
1440
1441Register fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, Register Op0) {
1442switch (RetVT.SimpleTy) {
1443 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0);
1444 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0);
1445 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0);
1446 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0);
1447 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0);
1448 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Op0);
1449 default: return Register();
1450}
1451}
1452
1453Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Register Op0) {
1454 if ((!Subtarget->isLittleEndian())) {
1455 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1456 }
1457 return Register();
1458}
1459
1460Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Register Op0) {
1461 if ((!Subtarget->isLittleEndian())) {
1462 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1463 }
1464 return Register();
1465}
1466
1467Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Register Op0) {
1468 if ((!Subtarget->isLittleEndian())) {
1469 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1470 }
1471 return Register();
1472}
1473
1474Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Register Op0) {
1475 if ((!Subtarget->isLittleEndian())) {
1476 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1477 }
1478 return Register();
1479}
1480
1481Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Register Op0) {
1482 if ((!Subtarget->isLittleEndian())) {
1483 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1484 }
1485 return Register();
1486}
1487
1488Register fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, Register Op0) {
1489switch (RetVT.SimpleTy) {
1490 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0);
1491 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0);
1492 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0);
1493 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0);
1494 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0);
1495 default: return Register();
1496}
1497}
1498
1499Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Register Op0) {
1500 if ((!Subtarget->isLittleEndian())) {
1501 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1502 }
1503 return Register();
1504}
1505
1506Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Register Op0) {
1507 if ((!Subtarget->isLittleEndian())) {
1508 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1509 }
1510 return Register();
1511}
1512
1513Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Register Op0) {
1514 if ((!Subtarget->isLittleEndian())) {
1515 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1516 }
1517 return Register();
1518}
1519
1520Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Register Op0) {
1521 if ((!Subtarget->isLittleEndian())) {
1522 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1523 }
1524 return Register();
1525}
1526
1527Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Register Op0) {
1528 if ((!Subtarget->isLittleEndian())) {
1529 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1530 }
1531 return Register();
1532}
1533
1534Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Register Op0) {
1535 if ((!Subtarget->isLittleEndian())) {
1536 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1537 }
1538 return Register();
1539}
1540
1541Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Register Op0) {
1542 if ((!Subtarget->isLittleEndian())) {
1543 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1544 }
1545 return Register();
1546}
1547
1548Register fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, Register Op0) {
1549switch (RetVT.SimpleTy) {
1550 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0);
1551 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0);
1552 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0);
1553 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0);
1554 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0);
1555 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Op0);
1556 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Op0);
1557 default: return Register();
1558}
1559}
1560
1561Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Register Op0) {
1562 if ((!Subtarget->isLittleEndian())) {
1563 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1564 }
1565 return Register();
1566}
1567
1568Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Register Op0) {
1569 if ((!Subtarget->isLittleEndian())) {
1570 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1571 }
1572 return Register();
1573}
1574
1575Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Register Op0) {
1576 if ((!Subtarget->isLittleEndian())) {
1577 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1578 }
1579 return Register();
1580}
1581
1582Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Register Op0) {
1583 if ((!Subtarget->isLittleEndian())) {
1584 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1585 }
1586 return Register();
1587}
1588
1589Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Register Op0) {
1590 if ((!Subtarget->isLittleEndian())) {
1591 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1592 }
1593 return Register();
1594}
1595
1596Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Register Op0) {
1597 if ((!Subtarget->isLittleEndian())) {
1598 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1599 }
1600 return Register();
1601}
1602
1603Register fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, Register Op0) {
1604switch (RetVT.SimpleTy) {
1605 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0);
1606 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0);
1607 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0);
1608 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0);
1609 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Op0);
1610 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0);
1611 default: return Register();
1612}
1613}
1614
1615Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Register Op0) {
1616 if ((!Subtarget->isLittleEndian())) {
1617 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1618 }
1619 return Register();
1620}
1621
1622Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Register Op0) {
1623 if ((!Subtarget->isLittleEndian())) {
1624 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1625 }
1626 return Register();
1627}
1628
1629Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Register Op0) {
1630 if ((!Subtarget->isLittleEndian())) {
1631 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1632 }
1633 return Register();
1634}
1635
1636Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Register Op0) {
1637 if ((!Subtarget->isLittleEndian())) {
1638 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1639 }
1640 return Register();
1641}
1642
1643Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Register Op0) {
1644 if ((!Subtarget->isLittleEndian())) {
1645 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1646 }
1647 return Register();
1648}
1649
1650Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Register Op0) {
1651 if ((!Subtarget->isLittleEndian())) {
1652 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1653 }
1654 return Register();
1655}
1656
1657Register fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, Register Op0) {
1658switch (RetVT.SimpleTy) {
1659 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0);
1660 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0);
1661 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0);
1662 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0);
1663 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Op0);
1664 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0);
1665 default: return Register();
1666}
1667}
1668
1669Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Register Op0) {
1670 if ((!Subtarget->isLittleEndian())) {
1671 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1672 }
1673 return Register();
1674}
1675
1676Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Register Op0) {
1677 if ((!Subtarget->isLittleEndian())) {
1678 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1679 }
1680 return Register();
1681}
1682
1683Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Register Op0) {
1684 if ((!Subtarget->isLittleEndian())) {
1685 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1686 }
1687 return Register();
1688}
1689
1690Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Register Op0) {
1691 if ((!Subtarget->isLittleEndian())) {
1692 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1693 }
1694 return Register();
1695}
1696
1697Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Register Op0) {
1698 if ((!Subtarget->isLittleEndian())) {
1699 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1700 }
1701 return Register();
1702}
1703
1704Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Register Op0) {
1705 if ((!Subtarget->isLittleEndian())) {
1706 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1707 }
1708 return Register();
1709}
1710
1711Register fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, Register Op0) {
1712switch (RetVT.SimpleTy) {
1713 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0);
1714 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0);
1715 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0);
1716 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0);
1717 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Op0);
1718 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0);
1719 default: return Register();
1720}
1721}
1722
1723Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Register Op0) {
1724 if ((!Subtarget->isLittleEndian())) {
1725 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1726 }
1727 return Register();
1728}
1729
1730Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Register Op0) {
1731 if ((!Subtarget->isLittleEndian())) {
1732 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1733 }
1734 return Register();
1735}
1736
1737Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Register Op0) {
1738 if ((!Subtarget->isLittleEndian())) {
1739 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1740 }
1741 return Register();
1742}
1743
1744Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Register Op0) {
1745 if ((!Subtarget->isLittleEndian())) {
1746 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1747 }
1748 return Register();
1749}
1750
1751Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Register Op0) {
1752 if ((!Subtarget->isLittleEndian())) {
1753 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1754 }
1755 return Register();
1756}
1757
1758Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Register Op0) {
1759 if ((!Subtarget->isLittleEndian())) {
1760 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1761 }
1762 return Register();
1763}
1764
1765Register fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, Register Op0) {
1766switch (RetVT.SimpleTy) {
1767 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0);
1768 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0);
1769 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0);
1770 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0);
1771 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0);
1772 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Op0);
1773 default: return Register();
1774}
1775}
1776
1777Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Register Op0) {
1778 if ((!Subtarget->isLittleEndian())) {
1779 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1780 }
1781 return Register();
1782}
1783
1784Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Register Op0) {
1785 if ((!Subtarget->isLittleEndian())) {
1786 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1787 }
1788 return Register();
1789}
1790
1791Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Register Op0) {
1792 if ((!Subtarget->isLittleEndian())) {
1793 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1794 }
1795 return Register();
1796}
1797
1798Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Register Op0) {
1799 if ((!Subtarget->isLittleEndian())) {
1800 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1801 }
1802 return Register();
1803}
1804
1805Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Register Op0) {
1806 if ((!Subtarget->isLittleEndian())) {
1807 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1808 }
1809 return Register();
1810}
1811
1812Register fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, Register Op0) {
1813switch (RetVT.SimpleTy) {
1814 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0);
1815 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0);
1816 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0);
1817 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0);
1818 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0);
1819 default: return Register();
1820}
1821}
1822
1823Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Register Op0) {
1824 if ((!Subtarget->isLittleEndian())) {
1825 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1826 }
1827 return Register();
1828}
1829
1830Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Register Op0) {
1831 if ((!Subtarget->isLittleEndian())) {
1832 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1833 }
1834 return Register();
1835}
1836
1837Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Register Op0) {
1838 if ((!Subtarget->isLittleEndian())) {
1839 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1840 }
1841 return Register();
1842}
1843
1844Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Register Op0) {
1845 if ((!Subtarget->isLittleEndian())) {
1846 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1847 }
1848 return Register();
1849}
1850
1851Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Register Op0) {
1852 if ((!Subtarget->isLittleEndian())) {
1853 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1854 }
1855 return Register();
1856}
1857
1858Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(Register Op0) {
1859 if ((!Subtarget->isLittleEndian())) {
1860 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1861 }
1862 return Register();
1863}
1864
1865Register fastEmit_ISD_BITCAST_MVT_v4bf16_r(MVT RetVT, Register Op0) {
1866switch (RetVT.SimpleTy) {
1867 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Op0);
1868 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Op0);
1869 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Op0);
1870 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Op0);
1871 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Op0);
1872 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(Op0);
1873 default: return Register();
1874}
1875}
1876
1877Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Register Op0) {
1878 if ((!Subtarget->isLittleEndian())) {
1879 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1880 }
1881 return Register();
1882}
1883
1884Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Register Op0) {
1885 if ((!Subtarget->isLittleEndian())) {
1886 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1887 }
1888 return Register();
1889}
1890
1891Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Register Op0) {
1892 if ((!Subtarget->isLittleEndian())) {
1893 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1894 }
1895 return Register();
1896}
1897
1898Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Register Op0) {
1899 if ((!Subtarget->isLittleEndian())) {
1900 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1901 }
1902 return Register();
1903}
1904
1905Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Register Op0) {
1906 if ((!Subtarget->isLittleEndian())) {
1907 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1908 }
1909 return Register();
1910}
1911
1912Register fastEmit_ISD_BITCAST_MVT_v8bf16_r(MVT RetVT, Register Op0) {
1913switch (RetVT.SimpleTy) {
1914 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Op0);
1915 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Op0);
1916 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Op0);
1917 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Op0);
1918 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Op0);
1919 default: return Register();
1920}
1921}
1922
1923Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Register Op0) {
1924 if ((!Subtarget->isLittleEndian())) {
1925 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1926 }
1927 return Register();
1928}
1929
1930Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Register Op0) {
1931 if ((!Subtarget->isLittleEndian())) {
1932 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1933 }
1934 return Register();
1935}
1936
1937Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Register Op0) {
1938 if ((!Subtarget->isLittleEndian())) {
1939 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1940 }
1941 return Register();
1942}
1943
1944Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Register Op0) {
1945 if ((!Subtarget->isLittleEndian())) {
1946 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1947 }
1948 return Register();
1949}
1950
1951Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Register Op0) {
1952 if ((!Subtarget->isLittleEndian())) {
1953 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1954 }
1955 return Register();
1956}
1957
1958Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Register Op0) {
1959 if ((!Subtarget->isLittleEndian())) {
1960 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1961 }
1962 return Register();
1963}
1964
1965Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Register Op0) {
1966 if ((!Subtarget->isLittleEndian())) {
1967 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1968 }
1969 return Register();
1970}
1971
1972Register fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, Register Op0) {
1973switch (RetVT.SimpleTy) {
1974 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0);
1975 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0);
1976 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0);
1977 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0);
1978 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0);
1979 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Op0);
1980 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Op0);
1981 default: return Register();
1982}
1983}
1984
1985Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Register Op0) {
1986 if ((!Subtarget->isLittleEndian())) {
1987 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1988 }
1989 return Register();
1990}
1991
1992Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Register Op0) {
1993 if ((!Subtarget->isLittleEndian())) {
1994 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1995 }
1996 return Register();
1997}
1998
1999Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Register Op0) {
2000 if ((!Subtarget->isLittleEndian())) {
2001 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2002 }
2003 return Register();
2004}
2005
2006Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Register Op0) {
2007 if ((!Subtarget->isLittleEndian())) {
2008 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2009 }
2010 return Register();
2011}
2012
2013Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Register Op0) {
2014 if ((!Subtarget->isLittleEndian())) {
2015 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2016 }
2017 return Register();
2018}
2019
2020Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Register Op0) {
2021 if ((!Subtarget->isLittleEndian())) {
2022 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2023 }
2024 return Register();
2025}
2026
2027Register fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, Register Op0) {
2028switch (RetVT.SimpleTy) {
2029 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0);
2030 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0);
2031 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0);
2032 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0);
2033 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Op0);
2034 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0);
2035 default: return Register();
2036}
2037}
2038
2039Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Register Op0) {
2040 if ((!Subtarget->isLittleEndian())) {
2041 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
2042 }
2043 return Register();
2044}
2045
2046Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Register Op0) {
2047 if ((!Subtarget->isLittleEndian())) {
2048 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2049 }
2050 return Register();
2051}
2052
2053Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Register Op0) {
2054 if ((!Subtarget->isLittleEndian())) {
2055 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2056 }
2057 return Register();
2058}
2059
2060Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Register Op0) {
2061 if ((!Subtarget->isLittleEndian())) {
2062 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2063 }
2064 return Register();
2065}
2066
2067Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(Register Op0) {
2068 if ((!Subtarget->isLittleEndian())) {
2069 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2070 }
2071 return Register();
2072}
2073
2074Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Register Op0) {
2075 if ((!Subtarget->isLittleEndian())) {
2076 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2077 }
2078 return Register();
2079}
2080
2081Register fastEmit_ISD_BITCAST_MVT_v1f64_r(MVT RetVT, Register Op0) {
2082switch (RetVT.SimpleTy) {
2083 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Op0);
2084 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Op0);
2085 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Op0);
2086 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Op0);
2087 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(Op0);
2088 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Op0);
2089 default: return Register();
2090}
2091}
2092
2093Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Register Op0) {
2094 if ((!Subtarget->isLittleEndian())) {
2095 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2096 }
2097 return Register();
2098}
2099
2100Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Register Op0) {
2101 if ((!Subtarget->isLittleEndian())) {
2102 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2103 }
2104 return Register();
2105}
2106
2107Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Register Op0) {
2108 if ((!Subtarget->isLittleEndian())) {
2109 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2110 }
2111 return Register();
2112}
2113
2114Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Register Op0) {
2115 if ((!Subtarget->isLittleEndian())) {
2116 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2117 }
2118 return Register();
2119}
2120
2121Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Register Op0) {
2122 if ((!Subtarget->isLittleEndian())) {
2123 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2124 }
2125 return Register();
2126}
2127
2128Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Register Op0) {
2129 if ((!Subtarget->isLittleEndian())) {
2130 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2131 }
2132 return Register();
2133}
2134
2135Register fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, Register Op0) {
2136switch (RetVT.SimpleTy) {
2137 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0);
2138 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0);
2139 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0);
2140 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0);
2141 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Op0);
2142 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0);
2143 default: return Register();
2144}
2145}
2146
2147Register fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, Register Op0) {
2148 switch (VT.SimpleTy) {
2149 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
2150 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0);
2151 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0);
2152 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0);
2153 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0);
2154 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0);
2155 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0);
2156 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0);
2157 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0);
2158 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0);
2159 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0);
2160 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v4bf16_r(RetVT, Op0);
2161 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v8bf16_r(RetVT, Op0);
2162 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0);
2163 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0);
2164 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v1f64_r(RetVT, Op0);
2165 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0);
2166 default: return Register();
2167 }
2168}
2169
2170// FastEmit functions for ISD::BITREVERSE.
2171
2172Register fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, Register Op0) {
2173 if (RetVT.SimpleTy != MVT::i32)
2174 return Register();
2175 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITWr, RC: &AArch64::GPR32RegClass, Op0);
2176}
2177
2178Register fastEmit_ISD_BITREVERSE_MVT_i64_r(MVT RetVT, Register Op0) {
2179 if (RetVT.SimpleTy != MVT::i64)
2180 return Register();
2181 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITXr, RC: &AArch64::GPR64RegClass, Op0);
2182}
2183
2184Register fastEmit_ISD_BITREVERSE_MVT_v8i8_r(MVT RetVT, Register Op0) {
2185 if (RetVT.SimpleTy != MVT::v8i8)
2186 return Register();
2187 if ((Subtarget->isNeonAvailable())) {
2188 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITv8i8, RC: &AArch64::FPR64RegClass, Op0);
2189 }
2190 return Register();
2191}
2192
2193Register fastEmit_ISD_BITREVERSE_MVT_v16i8_r(MVT RetVT, Register Op0) {
2194 if (RetVT.SimpleTy != MVT::v16i8)
2195 return Register();
2196 if ((Subtarget->isNeonAvailable())) {
2197 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITv16i8, RC: &AArch64::FPR128RegClass, Op0);
2198 }
2199 return Register();
2200}
2201
2202Register fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, Register Op0) {
2203 switch (VT.SimpleTy) {
2204 case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0);
2205 case MVT::i64: return fastEmit_ISD_BITREVERSE_MVT_i64_r(RetVT, Op0);
2206 case MVT::v8i8: return fastEmit_ISD_BITREVERSE_MVT_v8i8_r(RetVT, Op0);
2207 case MVT::v16i8: return fastEmit_ISD_BITREVERSE_MVT_v16i8_r(RetVT, Op0);
2208 default: return Register();
2209 }
2210}
2211
2212// FastEmit functions for ISD::BSWAP.
2213
2214Register fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, Register Op0) {
2215 if (RetVT.SimpleTy != MVT::i32)
2216 return Register();
2217 return fastEmitInst_r(MachineInstOpcode: AArch64::REVWr, RC: &AArch64::GPR32RegClass, Op0);
2218}
2219
2220Register fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, Register Op0) {
2221 if (RetVT.SimpleTy != MVT::i64)
2222 return Register();
2223 return fastEmitInst_r(MachineInstOpcode: AArch64::REVXr, RC: &AArch64::GPR64RegClass, Op0);
2224}
2225
2226Register fastEmit_ISD_BSWAP_MVT_v4i16_r(MVT RetVT, Register Op0) {
2227 if (RetVT.SimpleTy != MVT::v4i16)
2228 return Register();
2229 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
2230}
2231
2232Register fastEmit_ISD_BSWAP_MVT_v8i16_r(MVT RetVT, Register Op0) {
2233 if (RetVT.SimpleTy != MVT::v8i16)
2234 return Register();
2235 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
2236}
2237
2238Register fastEmit_ISD_BSWAP_MVT_v2i32_r(MVT RetVT, Register Op0) {
2239 if (RetVT.SimpleTy != MVT::v2i32)
2240 return Register();
2241 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
2242}
2243
2244Register fastEmit_ISD_BSWAP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2245 if (RetVT.SimpleTy != MVT::v4i32)
2246 return Register();
2247 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
2248}
2249
2250Register fastEmit_ISD_BSWAP_MVT_v2i64_r(MVT RetVT, Register Op0) {
2251 if (RetVT.SimpleTy != MVT::v2i64)
2252 return Register();
2253 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2254}
2255
2256Register fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, Register Op0) {
2257 switch (VT.SimpleTy) {
2258 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
2259 case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0);
2260 case MVT::v4i16: return fastEmit_ISD_BSWAP_MVT_v4i16_r(RetVT, Op0);
2261 case MVT::v8i16: return fastEmit_ISD_BSWAP_MVT_v8i16_r(RetVT, Op0);
2262 case MVT::v2i32: return fastEmit_ISD_BSWAP_MVT_v2i32_r(RetVT, Op0);
2263 case MVT::v4i32: return fastEmit_ISD_BSWAP_MVT_v4i32_r(RetVT, Op0);
2264 case MVT::v2i64: return fastEmit_ISD_BSWAP_MVT_v2i64_r(RetVT, Op0);
2265 default: return Register();
2266 }
2267}
2268
2269// FastEmit functions for ISD::CTLS.
2270
2271Register fastEmit_ISD_CTLS_MVT_i32_r(MVT RetVT, Register Op0) {
2272 if (RetVT.SimpleTy != MVT::i32)
2273 return Register();
2274 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSWr, RC: &AArch64::GPR32RegClass, Op0);
2275}
2276
2277Register fastEmit_ISD_CTLS_MVT_i64_r(MVT RetVT, Register Op0) {
2278 if (RetVT.SimpleTy != MVT::i64)
2279 return Register();
2280 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSXr, RC: &AArch64::GPR64RegClass, Op0);
2281}
2282
2283Register fastEmit_ISD_CTLS_MVT_v8i8_r(MVT RetVT, Register Op0) {
2284 if (RetVT.SimpleTy != MVT::v8i8)
2285 return Register();
2286 if ((Subtarget->isNeonAvailable())) {
2287 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv8i8, RC: &AArch64::FPR64RegClass, Op0);
2288 }
2289 return Register();
2290}
2291
2292Register fastEmit_ISD_CTLS_MVT_v16i8_r(MVT RetVT, Register Op0) {
2293 if (RetVT.SimpleTy != MVT::v16i8)
2294 return Register();
2295 if ((Subtarget->isNeonAvailable())) {
2296 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv16i8, RC: &AArch64::FPR128RegClass, Op0);
2297 }
2298 return Register();
2299}
2300
2301Register fastEmit_ISD_CTLS_MVT_v4i16_r(MVT RetVT, Register Op0) {
2302 if (RetVT.SimpleTy != MVT::v4i16)
2303 return Register();
2304 if ((Subtarget->isNeonAvailable())) {
2305 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv4i16, RC: &AArch64::FPR64RegClass, Op0);
2306 }
2307 return Register();
2308}
2309
2310Register fastEmit_ISD_CTLS_MVT_v8i16_r(MVT RetVT, Register Op0) {
2311 if (RetVT.SimpleTy != MVT::v8i16)
2312 return Register();
2313 if ((Subtarget->isNeonAvailable())) {
2314 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv8i16, RC: &AArch64::FPR128RegClass, Op0);
2315 }
2316 return Register();
2317}
2318
2319Register fastEmit_ISD_CTLS_MVT_v2i32_r(MVT RetVT, Register Op0) {
2320 if (RetVT.SimpleTy != MVT::v2i32)
2321 return Register();
2322 if ((Subtarget->isNeonAvailable())) {
2323 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv2i32, RC: &AArch64::FPR64RegClass, Op0);
2324 }
2325 return Register();
2326}
2327
2328Register fastEmit_ISD_CTLS_MVT_v4i32_r(MVT RetVT, Register Op0) {
2329 if (RetVT.SimpleTy != MVT::v4i32)
2330 return Register();
2331 if ((Subtarget->isNeonAvailable())) {
2332 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv4i32, RC: &AArch64::FPR128RegClass, Op0);
2333 }
2334 return Register();
2335}
2336
2337Register fastEmit_ISD_CTLS_r(MVT VT, MVT RetVT, Register Op0) {
2338 switch (VT.SimpleTy) {
2339 case MVT::i32: return fastEmit_ISD_CTLS_MVT_i32_r(RetVT, Op0);
2340 case MVT::i64: return fastEmit_ISD_CTLS_MVT_i64_r(RetVT, Op0);
2341 case MVT::v8i8: return fastEmit_ISD_CTLS_MVT_v8i8_r(RetVT, Op0);
2342 case MVT::v16i8: return fastEmit_ISD_CTLS_MVT_v16i8_r(RetVT, Op0);
2343 case MVT::v4i16: return fastEmit_ISD_CTLS_MVT_v4i16_r(RetVT, Op0);
2344 case MVT::v8i16: return fastEmit_ISD_CTLS_MVT_v8i16_r(RetVT, Op0);
2345 case MVT::v2i32: return fastEmit_ISD_CTLS_MVT_v2i32_r(RetVT, Op0);
2346 case MVT::v4i32: return fastEmit_ISD_CTLS_MVT_v4i32_r(RetVT, Op0);
2347 default: return Register();
2348 }
2349}
2350
2351// FastEmit functions for ISD::CTLZ.
2352
2353Register fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, Register Op0) {
2354 if (RetVT.SimpleTy != MVT::i32)
2355 return Register();
2356 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZWr, RC: &AArch64::GPR32RegClass, Op0);
2357}
2358
2359Register fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, Register Op0) {
2360 if (RetVT.SimpleTy != MVT::i64)
2361 return Register();
2362 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZXr, RC: &AArch64::GPR64RegClass, Op0);
2363}
2364
2365Register fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, Register Op0) {
2366 if (RetVT.SimpleTy != MVT::v8i8)
2367 return Register();
2368 if ((Subtarget->isNeonAvailable())) {
2369 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv8i8, RC: &AArch64::FPR64RegClass, Op0);
2370 }
2371 return Register();
2372}
2373
2374Register fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, Register Op0) {
2375 if (RetVT.SimpleTy != MVT::v16i8)
2376 return Register();
2377 if ((Subtarget->isNeonAvailable())) {
2378 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv16i8, RC: &AArch64::FPR128RegClass, Op0);
2379 }
2380 return Register();
2381}
2382
2383Register fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, Register Op0) {
2384 if (RetVT.SimpleTy != MVT::v4i16)
2385 return Register();
2386 if ((Subtarget->isNeonAvailable())) {
2387 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv4i16, RC: &AArch64::FPR64RegClass, Op0);
2388 }
2389 return Register();
2390}
2391
2392Register fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, Register Op0) {
2393 if (RetVT.SimpleTy != MVT::v8i16)
2394 return Register();
2395 if ((Subtarget->isNeonAvailable())) {
2396 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv8i16, RC: &AArch64::FPR128RegClass, Op0);
2397 }
2398 return Register();
2399}
2400
2401Register fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, Register Op0) {
2402 if (RetVT.SimpleTy != MVT::v2i32)
2403 return Register();
2404 if ((Subtarget->isNeonAvailable())) {
2405 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv2i32, RC: &AArch64::FPR64RegClass, Op0);
2406 }
2407 return Register();
2408}
2409
2410Register fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, Register Op0) {
2411 if (RetVT.SimpleTy != MVT::v4i32)
2412 return Register();
2413 if ((Subtarget->isNeonAvailable())) {
2414 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv4i32, RC: &AArch64::FPR128RegClass, Op0);
2415 }
2416 return Register();
2417}
2418
2419Register fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, Register Op0) {
2420 switch (VT.SimpleTy) {
2421 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
2422 case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0);
2423 case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0);
2424 case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0);
2425 case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0);
2426 case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0);
2427 case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0);
2428 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
2429 default: return Register();
2430 }
2431}
2432
2433// FastEmit functions for ISD::CTPOP.
2434
2435Register fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, Register Op0) {
2436 if (RetVT.SimpleTy != MVT::i32)
2437 return Register();
2438 if ((Subtarget->hasCSSC())) {
2439 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTWr, RC: &AArch64::GPR32RegClass, Op0);
2440 }
2441 return Register();
2442}
2443
2444Register fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, Register Op0) {
2445 if (RetVT.SimpleTy != MVT::i64)
2446 return Register();
2447 if ((Subtarget->hasCSSC())) {
2448 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTXr, RC: &AArch64::GPR64RegClass, Op0);
2449 }
2450 return Register();
2451}
2452
2453Register fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, Register Op0) {
2454 if (RetVT.SimpleTy != MVT::v8i8)
2455 return Register();
2456 if ((Subtarget->isNeonAvailable())) {
2457 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTv8i8, RC: &AArch64::FPR64RegClass, Op0);
2458 }
2459 return Register();
2460}
2461
2462Register fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, Register Op0) {
2463 if (RetVT.SimpleTy != MVT::v16i8)
2464 return Register();
2465 if ((Subtarget->isNeonAvailable())) {
2466 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTv16i8, RC: &AArch64::FPR128RegClass, Op0);
2467 }
2468 return Register();
2469}
2470
2471Register fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, Register Op0) {
2472 switch (VT.SimpleTy) {
2473 case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0);
2474 case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0);
2475 case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0);
2476 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
2477 default: return Register();
2478 }
2479}
2480
2481// FastEmit functions for ISD::CTTZ.
2482
2483Register fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, Register Op0) {
2484 if (RetVT.SimpleTy != MVT::i32)
2485 return Register();
2486 if ((Subtarget->hasCSSC())) {
2487 return fastEmitInst_r(MachineInstOpcode: AArch64::CTZWr, RC: &AArch64::GPR32RegClass, Op0);
2488 }
2489 return Register();
2490}
2491
2492Register fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, Register Op0) {
2493 if (RetVT.SimpleTy != MVT::i64)
2494 return Register();
2495 if ((Subtarget->hasCSSC())) {
2496 return fastEmitInst_r(MachineInstOpcode: AArch64::CTZXr, RC: &AArch64::GPR64RegClass, Op0);
2497 }
2498 return Register();
2499}
2500
2501Register fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, Register Op0) {
2502 switch (VT.SimpleTy) {
2503 case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0);
2504 case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0);
2505 default: return Register();
2506 }
2507}
2508
2509// FastEmit functions for ISD::FABS.
2510
2511Register fastEmit_ISD_FABS_MVT_bf16_r(MVT RetVT, Register Op0) {
2512 if (RetVT.SimpleTy != MVT::bf16)
2513 return Register();
2514 if ((Subtarget->hasFullFP16())) {
2515 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSHr, RC: &AArch64::FPR16RegClass, Op0);
2516 }
2517 return Register();
2518}
2519
2520Register fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, Register Op0) {
2521 if (RetVT.SimpleTy != MVT::f16)
2522 return Register();
2523 if ((Subtarget->hasFullFP16())) {
2524 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSHr, RC: &AArch64::FPR16RegClass, Op0);
2525 }
2526 return Register();
2527}
2528
2529Register fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, Register Op0) {
2530 if (RetVT.SimpleTy != MVT::f32)
2531 return Register();
2532 if ((Subtarget->hasFPARMv8())) {
2533 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSSr, RC: &AArch64::FPR32RegClass, Op0);
2534 }
2535 return Register();
2536}
2537
2538Register fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, Register Op0) {
2539 if (RetVT.SimpleTy != MVT::f64)
2540 return Register();
2541 if ((Subtarget->hasFPARMv8())) {
2542 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSDr, RC: &AArch64::FPR64RegClass, Op0);
2543 }
2544 return Register();
2545}
2546
2547Register fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, Register Op0) {
2548 if (RetVT.SimpleTy != MVT::v4f16)
2549 return Register();
2550 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2551 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f16, RC: &AArch64::FPR64RegClass, Op0);
2552 }
2553 return Register();
2554}
2555
2556Register fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, Register Op0) {
2557 if (RetVT.SimpleTy != MVT::v8f16)
2558 return Register();
2559 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2560 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv8f16, RC: &AArch64::FPR128RegClass, Op0);
2561 }
2562 return Register();
2563}
2564
2565Register fastEmit_ISD_FABS_MVT_v4bf16_r(MVT RetVT, Register Op0) {
2566 if (RetVT.SimpleTy != MVT::v4bf16)
2567 return Register();
2568 if ((Subtarget->hasFullFP16())) {
2569 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f16, RC: &AArch64::FPR64RegClass, Op0);
2570 }
2571 return Register();
2572}
2573
2574Register fastEmit_ISD_FABS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
2575 if (RetVT.SimpleTy != MVT::v8bf16)
2576 return Register();
2577 if ((Subtarget->hasFullFP16())) {
2578 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv8f16, RC: &AArch64::FPR128RegClass, Op0);
2579 }
2580 return Register();
2581}
2582
2583Register fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, Register Op0) {
2584 if (RetVT.SimpleTy != MVT::v2f32)
2585 return Register();
2586 if ((Subtarget->isNeonAvailable())) {
2587 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv2f32, RC: &AArch64::FPR64RegClass, Op0);
2588 }
2589 return Register();
2590}
2591
2592Register fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, Register Op0) {
2593 if (RetVT.SimpleTy != MVT::v4f32)
2594 return Register();
2595 if ((Subtarget->isNeonAvailable())) {
2596 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f32, RC: &AArch64::FPR128RegClass, Op0);
2597 }
2598 return Register();
2599}
2600
2601Register fastEmit_ISD_FABS_MVT_v2f64_r(MVT RetVT, Register Op0) {
2602 if (RetVT.SimpleTy != MVT::v2f64)
2603 return Register();
2604 if ((Subtarget->isNeonAvailable())) {
2605 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv2f64, RC: &AArch64::FPR128RegClass, Op0);
2606 }
2607 return Register();
2608}
2609
2610Register fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, Register Op0) {
2611 switch (VT.SimpleTy) {
2612 case MVT::bf16: return fastEmit_ISD_FABS_MVT_bf16_r(RetVT, Op0);
2613 case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0);
2614 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
2615 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
2616 case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0);
2617 case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0);
2618 case MVT::v4bf16: return fastEmit_ISD_FABS_MVT_v4bf16_r(RetVT, Op0);
2619 case MVT::v8bf16: return fastEmit_ISD_FABS_MVT_v8bf16_r(RetVT, Op0);
2620 case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0);
2621 case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0);
2622 case MVT::v2f64: return fastEmit_ISD_FABS_MVT_v2f64_r(RetVT, Op0);
2623 default: return Register();
2624 }
2625}
2626
2627// FastEmit functions for ISD::FCEIL.
2628
2629Register fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
2630 if (RetVT.SimpleTy != MVT::f16)
2631 return Register();
2632 if ((Subtarget->hasFullFP16())) {
2633 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPHr, RC: &AArch64::FPR16RegClass, Op0);
2634 }
2635 return Register();
2636}
2637
2638Register fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
2639 if (RetVT.SimpleTy != MVT::f32)
2640 return Register();
2641 if ((Subtarget->hasFPARMv8())) {
2642 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPSr, RC: &AArch64::FPR32RegClass, Op0);
2643 }
2644 return Register();
2645}
2646
2647Register fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
2648 if (RetVT.SimpleTy != MVT::f64)
2649 return Register();
2650 if ((Subtarget->hasFPARMv8())) {
2651 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPDr, RC: &AArch64::FPR64RegClass, Op0);
2652 }
2653 return Register();
2654}
2655
2656Register fastEmit_ISD_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
2657 if (RetVT.SimpleTy != MVT::v4f16)
2658 return Register();
2659 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2660 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f16, RC: &AArch64::FPR64RegClass, Op0);
2661 }
2662 return Register();
2663}
2664
2665Register fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
2666 if (RetVT.SimpleTy != MVT::v8f16)
2667 return Register();
2668 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2669 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv8f16, RC: &AArch64::FPR128RegClass, Op0);
2670 }
2671 return Register();
2672}
2673
2674Register fastEmit_ISD_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
2675 if (RetVT.SimpleTy != MVT::v2f32)
2676 return Register();
2677 if ((Subtarget->isNeonAvailable())) {
2678 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f32, RC: &AArch64::FPR64RegClass, Op0);
2679 }
2680 return Register();
2681}
2682
2683Register fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
2684 if (RetVT.SimpleTy != MVT::v4f32)
2685 return Register();
2686 if ((Subtarget->isNeonAvailable())) {
2687 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f32, RC: &AArch64::FPR128RegClass, Op0);
2688 }
2689 return Register();
2690}
2691
2692Register fastEmit_ISD_FCEIL_MVT_v2f64_r(MVT RetVT, Register Op0) {
2693 if (RetVT.SimpleTy != MVT::v2f64)
2694 return Register();
2695 if ((Subtarget->isNeonAvailable())) {
2696 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f64, RC: &AArch64::FPR128RegClass, Op0);
2697 }
2698 return Register();
2699}
2700
2701Register fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
2702 switch (VT.SimpleTy) {
2703 case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0);
2704 case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0);
2705 case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0);
2706 case MVT::v4f16: return fastEmit_ISD_FCEIL_MVT_v4f16_r(RetVT, Op0);
2707 case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0);
2708 case MVT::v2f32: return fastEmit_ISD_FCEIL_MVT_v2f32_r(RetVT, Op0);
2709 case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0);
2710 case MVT::v2f64: return fastEmit_ISD_FCEIL_MVT_v2f64_r(RetVT, Op0);
2711 default: return Register();
2712 }
2713}
2714
2715// FastEmit functions for ISD::FFLOOR.
2716
2717Register fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
2718 if (RetVT.SimpleTy != MVT::f16)
2719 return Register();
2720 if ((Subtarget->hasFullFP16())) {
2721 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMHr, RC: &AArch64::FPR16RegClass, Op0);
2722 }
2723 return Register();
2724}
2725
2726Register fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
2727 if (RetVT.SimpleTy != MVT::f32)
2728 return Register();
2729 if ((Subtarget->hasFPARMv8())) {
2730 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMSr, RC: &AArch64::FPR32RegClass, Op0);
2731 }
2732 return Register();
2733}
2734
2735Register fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
2736 if (RetVT.SimpleTy != MVT::f64)
2737 return Register();
2738 if ((Subtarget->hasFPARMv8())) {
2739 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMDr, RC: &AArch64::FPR64RegClass, Op0);
2740 }
2741 return Register();
2742}
2743
2744Register fastEmit_ISD_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
2745 if (RetVT.SimpleTy != MVT::v4f16)
2746 return Register();
2747 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2748 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f16, RC: &AArch64::FPR64RegClass, Op0);
2749 }
2750 return Register();
2751}
2752
2753Register fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
2754 if (RetVT.SimpleTy != MVT::v8f16)
2755 return Register();
2756 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2757 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv8f16, RC: &AArch64::FPR128RegClass, Op0);
2758 }
2759 return Register();
2760}
2761
2762Register fastEmit_ISD_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
2763 if (RetVT.SimpleTy != MVT::v2f32)
2764 return Register();
2765 if ((Subtarget->isNeonAvailable())) {
2766 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f32, RC: &AArch64::FPR64RegClass, Op0);
2767 }
2768 return Register();
2769}
2770
2771Register fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
2772 if (RetVT.SimpleTy != MVT::v4f32)
2773 return Register();
2774 if ((Subtarget->isNeonAvailable())) {
2775 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f32, RC: &AArch64::FPR128RegClass, Op0);
2776 }
2777 return Register();
2778}
2779
2780Register fastEmit_ISD_FFLOOR_MVT_v2f64_r(MVT RetVT, Register Op0) {
2781 if (RetVT.SimpleTy != MVT::v2f64)
2782 return Register();
2783 if ((Subtarget->isNeonAvailable())) {
2784 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f64, RC: &AArch64::FPR128RegClass, Op0);
2785 }
2786 return Register();
2787}
2788
2789Register fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
2790 switch (VT.SimpleTy) {
2791 case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0);
2792 case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0);
2793 case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0);
2794 case MVT::v4f16: return fastEmit_ISD_FFLOOR_MVT_v4f16_r(RetVT, Op0);
2795 case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0);
2796 case MVT::v2f32: return fastEmit_ISD_FFLOOR_MVT_v2f32_r(RetVT, Op0);
2797 case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0);
2798 case MVT::v2f64: return fastEmit_ISD_FFLOOR_MVT_v2f64_r(RetVT, Op0);
2799 default: return Register();
2800 }
2801}
2802
2803// FastEmit functions for ISD::FNEARBYINT.
2804
2805Register fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
2806 if (RetVT.SimpleTy != MVT::f16)
2807 return Register();
2808 if ((Subtarget->hasFullFP16())) {
2809 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIHr, RC: &AArch64::FPR16RegClass, Op0);
2810 }
2811 return Register();
2812}
2813
2814Register fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
2815 if (RetVT.SimpleTy != MVT::f32)
2816 return Register();
2817 if ((Subtarget->hasFPARMv8())) {
2818 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTISr, RC: &AArch64::FPR32RegClass, Op0);
2819 }
2820 return Register();
2821}
2822
2823Register fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
2824 if (RetVT.SimpleTy != MVT::f64)
2825 return Register();
2826 if ((Subtarget->hasFPARMv8())) {
2827 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIDr, RC: &AArch64::FPR64RegClass, Op0);
2828 }
2829 return Register();
2830}
2831
2832Register fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2833 if (RetVT.SimpleTy != MVT::v4f16)
2834 return Register();
2835 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2836 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f16, RC: &AArch64::FPR64RegClass, Op0);
2837 }
2838 return Register();
2839}
2840
2841Register fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2842 if (RetVT.SimpleTy != MVT::v8f16)
2843 return Register();
2844 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2845 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv8f16, RC: &AArch64::FPR128RegClass, Op0);
2846 }
2847 return Register();
2848}
2849
2850Register fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2851 if (RetVT.SimpleTy != MVT::v2f32)
2852 return Register();
2853 if ((Subtarget->isNeonAvailable())) {
2854 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f32, RC: &AArch64::FPR64RegClass, Op0);
2855 }
2856 return Register();
2857}
2858
2859Register fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2860 if (RetVT.SimpleTy != MVT::v4f32)
2861 return Register();
2862 if ((Subtarget->isNeonAvailable())) {
2863 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f32, RC: &AArch64::FPR128RegClass, Op0);
2864 }
2865 return Register();
2866}
2867
2868Register fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
2869 if (RetVT.SimpleTy != MVT::v2f64)
2870 return Register();
2871 if ((Subtarget->isNeonAvailable())) {
2872 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f64, RC: &AArch64::FPR128RegClass, Op0);
2873 }
2874 return Register();
2875}
2876
2877Register fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
2878 switch (VT.SimpleTy) {
2879 case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0);
2880 case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0);
2881 case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0);
2882 case MVT::v4f16: return fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(RetVT, Op0);
2883 case MVT::v8f16: return fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(RetVT, Op0);
2884 case MVT::v2f32: return fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(RetVT, Op0);
2885 case MVT::v4f32: return fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(RetVT, Op0);
2886 case MVT::v2f64: return fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(RetVT, Op0);
2887 default: return Register();
2888 }
2889}
2890
2891// FastEmit functions for ISD::FNEG.
2892
2893Register fastEmit_ISD_FNEG_MVT_bf16_r(MVT RetVT, Register Op0) {
2894 if (RetVT.SimpleTy != MVT::bf16)
2895 return Register();
2896 if ((Subtarget->hasFullFP16())) {
2897 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGHr, RC: &AArch64::FPR16RegClass, Op0);
2898 }
2899 return Register();
2900}
2901
2902Register fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, Register Op0) {
2903 if (RetVT.SimpleTy != MVT::f16)
2904 return Register();
2905 if ((Subtarget->hasFullFP16())) {
2906 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGHr, RC: &AArch64::FPR16RegClass, Op0);
2907 }
2908 return Register();
2909}
2910
2911Register fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, Register Op0) {
2912 if (RetVT.SimpleTy != MVT::f32)
2913 return Register();
2914 if ((Subtarget->hasFPARMv8())) {
2915 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGSr, RC: &AArch64::FPR32RegClass, Op0);
2916 }
2917 return Register();
2918}
2919
2920Register fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, Register Op0) {
2921 if (RetVT.SimpleTy != MVT::f64)
2922 return Register();
2923 if ((Subtarget->hasFPARMv8())) {
2924 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGDr, RC: &AArch64::FPR64RegClass, Op0);
2925 }
2926 return Register();
2927}
2928
2929Register fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, Register Op0) {
2930 if (RetVT.SimpleTy != MVT::v4f16)
2931 return Register();
2932 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2933 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f16, RC: &AArch64::FPR64RegClass, Op0);
2934 }
2935 return Register();
2936}
2937
2938Register fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, Register Op0) {
2939 if (RetVT.SimpleTy != MVT::v8f16)
2940 return Register();
2941 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2942 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv8f16, RC: &AArch64::FPR128RegClass, Op0);
2943 }
2944 return Register();
2945}
2946
2947Register fastEmit_ISD_FNEG_MVT_v4bf16_r(MVT RetVT, Register Op0) {
2948 if (RetVT.SimpleTy != MVT::v4bf16)
2949 return Register();
2950 if ((Subtarget->hasFullFP16())) {
2951 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f16, RC: &AArch64::FPR64RegClass, Op0);
2952 }
2953 return Register();
2954}
2955
2956Register fastEmit_ISD_FNEG_MVT_v8bf16_r(MVT RetVT, Register Op0) {
2957 if (RetVT.SimpleTy != MVT::v8bf16)
2958 return Register();
2959 if ((Subtarget->hasFullFP16())) {
2960 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv8f16, RC: &AArch64::FPR128RegClass, Op0);
2961 }
2962 return Register();
2963}
2964
2965Register fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, Register Op0) {
2966 if (RetVT.SimpleTy != MVT::v2f32)
2967 return Register();
2968 if ((Subtarget->isNeonAvailable())) {
2969 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv2f32, RC: &AArch64::FPR64RegClass, Op0);
2970 }
2971 return Register();
2972}
2973
2974Register fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, Register Op0) {
2975 if (RetVT.SimpleTy != MVT::v4f32)
2976 return Register();
2977 if ((Subtarget->isNeonAvailable())) {
2978 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f32, RC: &AArch64::FPR128RegClass, Op0);
2979 }
2980 return Register();
2981}
2982
2983Register fastEmit_ISD_FNEG_MVT_v2f64_r(MVT RetVT, Register Op0) {
2984 if (RetVT.SimpleTy != MVT::v2f64)
2985 return Register();
2986 if ((Subtarget->isNeonAvailable())) {
2987 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv2f64, RC: &AArch64::FPR128RegClass, Op0);
2988 }
2989 return Register();
2990}
2991
2992Register fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, Register Op0) {
2993 switch (VT.SimpleTy) {
2994 case MVT::bf16: return fastEmit_ISD_FNEG_MVT_bf16_r(RetVT, Op0);
2995 case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0);
2996 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
2997 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
2998 case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0);
2999 case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0);
3000 case MVT::v4bf16: return fastEmit_ISD_FNEG_MVT_v4bf16_r(RetVT, Op0);
3001 case MVT::v8bf16: return fastEmit_ISD_FNEG_MVT_v8bf16_r(RetVT, Op0);
3002 case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0);
3003 case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0);
3004 case MVT::v2f64: return fastEmit_ISD_FNEG_MVT_v2f64_r(RetVT, Op0);
3005 default: return Register();
3006 }
3007}
3008
3009// FastEmit functions for ISD::FP_EXTEND.
3010
3011Register fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Register Op0) {
3012 if ((Subtarget->hasFPARMv8())) {
3013 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSHr, RC: &AArch64::FPR32RegClass, Op0);
3014 }
3015 return Register();
3016}
3017
3018Register fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Register Op0) {
3019 if ((Subtarget->hasFPARMv8())) {
3020 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDHr, RC: &AArch64::FPR64RegClass, Op0);
3021 }
3022 return Register();
3023}
3024
3025Register fastEmit_ISD_FP_EXTEND_MVT_f16_r(MVT RetVT, Register Op0) {
3026switch (RetVT.SimpleTy) {
3027 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Op0);
3028 case MVT::f64: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Op0);
3029 default: return Register();
3030}
3031}
3032
3033Register fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
3034 if (RetVT.SimpleTy != MVT::f64)
3035 return Register();
3036 if ((Subtarget->hasFPARMv8())) {
3037 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDSr, RC: &AArch64::FPR64RegClass, Op0);
3038 }
3039 return Register();
3040}
3041
3042Register fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3043 if (RetVT.SimpleTy != MVT::v4f32)
3044 return Register();
3045 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv4i16, RC: &AArch64::FPR128RegClass, Op0);
3046}
3047
3048Register fastEmit_ISD_FP_EXTEND_MVT_v4bf16_r(MVT RetVT, Register Op0) {
3049 if (RetVT.SimpleTy != MVT::v4f32)
3050 return Register();
3051 if ((Subtarget->isNeonAvailable())) {
3052 return fastEmitInst_r(MachineInstOpcode: AArch64::SHLLv4i16, RC: &AArch64::FPR128RegClass, Op0);
3053 }
3054 return Register();
3055}
3056
3057Register fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3058 if (RetVT.SimpleTy != MVT::v2f64)
3059 return Register();
3060 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv2i32, RC: &AArch64::FPR128RegClass, Op0);
3061}
3062
3063Register fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3064 switch (VT.SimpleTy) {
3065 case MVT::f16: return fastEmit_ISD_FP_EXTEND_MVT_f16_r(RetVT, Op0);
3066 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
3067 case MVT::v4f16: return fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
3068 case MVT::v4bf16: return fastEmit_ISD_FP_EXTEND_MVT_v4bf16_r(RetVT, Op0);
3069 case MVT::v2f32: return fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(RetVT, Op0);
3070 default: return Register();
3071 }
3072}
3073
3074// FastEmit functions for ISD::FP_ROUND.
3075
3076Register fastEmit_ISD_FP_ROUND_MVT_f32_MVT_bf16_r(Register Op0) {
3077 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
3078 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVT, RC: &AArch64::FPR16RegClass, Op0);
3079 }
3080 return Register();
3081}
3082
3083Register fastEmit_ISD_FP_ROUND_MVT_f32_MVT_f16_r(Register Op0) {
3084 if ((Subtarget->hasFPARMv8())) {
3085 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHSr, RC: &AArch64::FPR16RegClass, Op0);
3086 }
3087 return Register();
3088}
3089
3090Register fastEmit_ISD_FP_ROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3091switch (RetVT.SimpleTy) {
3092 case MVT::bf16: return fastEmit_ISD_FP_ROUND_MVT_f32_MVT_bf16_r(Op0);
3093 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f32_MVT_f16_r(Op0);
3094 default: return Register();
3095}
3096}
3097
3098Register fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Register Op0) {
3099 if ((Subtarget->hasFPARMv8())) {
3100 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHDr, RC: &AArch64::FPR16RegClass, Op0);
3101 }
3102 return Register();
3103}
3104
3105Register fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Register Op0) {
3106 if ((Subtarget->hasFPARMv8())) {
3107 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSDr, RC: &AArch64::FPR32RegClass, Op0);
3108 }
3109 return Register();
3110}
3111
3112Register fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3113switch (RetVT.SimpleTy) {
3114 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Op0);
3115 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Op0);
3116 default: return Register();
3117}
3118}
3119
3120Register fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Register Op0) {
3121 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
3122}
3123
3124Register fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Register Op0) {
3125 if ((Subtarget->hasBF16()) && (Subtarget->isNeonAvailable())) {
3126 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVTN, RC: &AArch64::FPR64RegClass, Op0);
3127 }
3128 return Register();
3129}
3130
3131Register fastEmit_ISD_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3132switch (RetVT.SimpleTy) {
3133 case MVT::v4f16: return fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Op0);
3134 case MVT::v4bf16: return fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Op0);
3135 default: return Register();
3136}
3137}
3138
3139Register fastEmit_ISD_FP_ROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
3140 if (RetVT.SimpleTy != MVT::v2f32)
3141 return Register();
3142 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
3143}
3144
3145Register fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
3146 switch (VT.SimpleTy) {
3147 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f32_r(RetVT, Op0);
3148 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
3149 case MVT::v4f32: return fastEmit_ISD_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
3150 case MVT::v2f64: return fastEmit_ISD_FP_ROUND_MVT_v2f64_r(RetVT, Op0);
3151 default: return Register();
3152 }
3153}
3154
3155// FastEmit functions for ISD::FP_TO_SINT.
3156
3157Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
3158 if ((Subtarget->hasFPRCVT())) {
3159 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSHr, RC: &AArch64::FPR32RegClass, Op0);
3160 }
3161 if ((Subtarget->hasFullFP16())) {
3162 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
3163 }
3164 return Register();
3165}
3166
3167Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
3168 if ((Subtarget->hasFPRCVT())) {
3169 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDHr, RC: &AArch64::FPR64RegClass, Op0);
3170 }
3171 if ((Subtarget->hasFullFP16())) {
3172 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
3173 }
3174 return Register();
3175}
3176
3177Register fastEmit_ISD_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
3178switch (RetVT.SimpleTy) {
3179 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
3180 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
3181 default: return Register();
3182}
3183}
3184
3185Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
3186 if ((Subtarget->hasFPARMv8())) {
3187 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
3188 }
3189 return Register();
3190}
3191
3192Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
3193 if ((Subtarget->hasFPRCVT())) {
3194 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDSr, RC: &AArch64::FPR64RegClass, Op0);
3195 }
3196 if ((Subtarget->hasFPARMv8())) {
3197 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
3198 }
3199 return Register();
3200}
3201
3202Register fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
3203switch (RetVT.SimpleTy) {
3204 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
3205 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
3206 default: return Register();
3207}
3208}
3209
3210Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
3211 if ((Subtarget->hasFPRCVT())) {
3212 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSDr, RC: &AArch64::FPR32RegClass, Op0);
3213 }
3214 if ((Subtarget->hasFPARMv8())) {
3215 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
3216 }
3217 return Register();
3218}
3219
3220Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
3221 if ((Subtarget->hasFPARMv8())) {
3222 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
3223 }
3224 return Register();
3225}
3226
3227Register fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
3228switch (RetVT.SimpleTy) {
3229 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
3230 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
3231 default: return Register();
3232}
3233}
3234
3235Register fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3236 if (RetVT.SimpleTy != MVT::v4i16)
3237 return Register();
3238 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3239 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
3240 }
3241 return Register();
3242}
3243
3244Register fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3245 if (RetVT.SimpleTy != MVT::v8i16)
3246 return Register();
3247 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3248 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
3249 }
3250 return Register();
3251}
3252
3253Register fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3254 if (RetVT.SimpleTy != MVT::v2i32)
3255 return Register();
3256 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3257 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
3258 }
3259 return Register();
3260}
3261
3262Register fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3263 if (RetVT.SimpleTy != MVT::v4i32)
3264 return Register();
3265 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3266 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
3267 }
3268 return Register();
3269}
3270
3271Register fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3272 if (RetVT.SimpleTy != MVT::v2i64)
3273 return Register();
3274 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3275 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
3276 }
3277 return Register();
3278}
3279
3280Register fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
3281 switch (VT.SimpleTy) {
3282 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
3283 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
3284 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
3285 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
3286 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
3287 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
3288 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
3289 case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0);
3290 default: return Register();
3291 }
3292}
3293
3294// FastEmit functions for ISD::FP_TO_UINT.
3295
3296Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
3297 if ((Subtarget->hasFPRCVT())) {
3298 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSHr, RC: &AArch64::FPR32RegClass, Op0);
3299 }
3300 if ((Subtarget->hasFullFP16())) {
3301 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
3302 }
3303 return Register();
3304}
3305
3306Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
3307 if ((Subtarget->hasFPRCVT())) {
3308 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDHr, RC: &AArch64::FPR64RegClass, Op0);
3309 }
3310 if ((Subtarget->hasFullFP16())) {
3311 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
3312 }
3313 return Register();
3314}
3315
3316Register fastEmit_ISD_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
3317switch (RetVT.SimpleTy) {
3318 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
3319 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
3320 default: return Register();
3321}
3322}
3323
3324Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
3325 if ((Subtarget->hasFPARMv8())) {
3326 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
3327 }
3328 return Register();
3329}
3330
3331Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
3332 if ((Subtarget->hasFPRCVT())) {
3333 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDSr, RC: &AArch64::FPR64RegClass, Op0);
3334 }
3335 if ((Subtarget->hasFPARMv8())) {
3336 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
3337 }
3338 return Register();
3339}
3340
3341Register fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
3342switch (RetVT.SimpleTy) {
3343 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
3344 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
3345 default: return Register();
3346}
3347}
3348
3349Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
3350 if ((Subtarget->hasFPRCVT())) {
3351 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSDr, RC: &AArch64::FPR32RegClass, Op0);
3352 }
3353 if ((Subtarget->hasFPARMv8())) {
3354 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
3355 }
3356 return Register();
3357}
3358
3359Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
3360 if ((Subtarget->hasFPARMv8())) {
3361 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
3362 }
3363 return Register();
3364}
3365
3366Register fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
3367switch (RetVT.SimpleTy) {
3368 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
3369 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
3370 default: return Register();
3371}
3372}
3373
3374Register fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3375 if (RetVT.SimpleTy != MVT::v4i16)
3376 return Register();
3377 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3378 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
3379 }
3380 return Register();
3381}
3382
3383Register fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3384 if (RetVT.SimpleTy != MVT::v8i16)
3385 return Register();
3386 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3387 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
3388 }
3389 return Register();
3390}
3391
3392Register fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3393 if (RetVT.SimpleTy != MVT::v2i32)
3394 return Register();
3395 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3396 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
3397 }
3398 return Register();
3399}
3400
3401Register fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3402 if (RetVT.SimpleTy != MVT::v4i32)
3403 return Register();
3404 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3405 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
3406 }
3407 return Register();
3408}
3409
3410Register fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3411 if (RetVT.SimpleTy != MVT::v2i64)
3412 return Register();
3413 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3414 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
3415 }
3416 return Register();
3417}
3418
3419Register fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
3420 switch (VT.SimpleTy) {
3421 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
3422 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
3423 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
3424 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
3425 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
3426 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
3427 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
3428 case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0);
3429 default: return Register();
3430 }
3431}
3432
3433// FastEmit functions for ISD::FRINT.
3434
3435Register fastEmit_ISD_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
3436 if (RetVT.SimpleTy != MVT::f16)
3437 return Register();
3438 if ((Subtarget->hasFullFP16())) {
3439 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXHr, RC: &AArch64::FPR16RegClass, Op0);
3440 }
3441 return Register();
3442}
3443
3444Register fastEmit_ISD_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
3445 if (RetVT.SimpleTy != MVT::f32)
3446 return Register();
3447 if ((Subtarget->hasFPARMv8())) {
3448 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXSr, RC: &AArch64::FPR32RegClass, Op0);
3449 }
3450 return Register();
3451}
3452
3453Register fastEmit_ISD_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
3454 if (RetVT.SimpleTy != MVT::f64)
3455 return Register();
3456 if ((Subtarget->hasFPARMv8())) {
3457 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXDr, RC: &AArch64::FPR64RegClass, Op0);
3458 }
3459 return Register();
3460}
3461
3462Register fastEmit_ISD_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3463 if (RetVT.SimpleTy != MVT::v4f16)
3464 return Register();
3465 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3466 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f16, RC: &AArch64::FPR64RegClass, Op0);
3467 }
3468 return Register();
3469}
3470
3471Register fastEmit_ISD_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3472 if (RetVT.SimpleTy != MVT::v8f16)
3473 return Register();
3474 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3475 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv8f16, RC: &AArch64::FPR128RegClass, Op0);
3476 }
3477 return Register();
3478}
3479
3480Register fastEmit_ISD_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3481 if (RetVT.SimpleTy != MVT::v2f32)
3482 return Register();
3483 if ((Subtarget->isNeonAvailable())) {
3484 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f32, RC: &AArch64::FPR64RegClass, Op0);
3485 }
3486 return Register();
3487}
3488
3489Register fastEmit_ISD_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3490 if (RetVT.SimpleTy != MVT::v4f32)
3491 return Register();
3492 if ((Subtarget->isNeonAvailable())) {
3493 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f32, RC: &AArch64::FPR128RegClass, Op0);
3494 }
3495 return Register();
3496}
3497
3498Register fastEmit_ISD_FRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3499 if (RetVT.SimpleTy != MVT::v2f64)
3500 return Register();
3501 if ((Subtarget->isNeonAvailable())) {
3502 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f64, RC: &AArch64::FPR128RegClass, Op0);
3503 }
3504 return Register();
3505}
3506
3507Register fastEmit_ISD_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
3508 switch (VT.SimpleTy) {
3509 case MVT::f16: return fastEmit_ISD_FRINT_MVT_f16_r(RetVT, Op0);
3510 case MVT::f32: return fastEmit_ISD_FRINT_MVT_f32_r(RetVT, Op0);
3511 case MVT::f64: return fastEmit_ISD_FRINT_MVT_f64_r(RetVT, Op0);
3512 case MVT::v4f16: return fastEmit_ISD_FRINT_MVT_v4f16_r(RetVT, Op0);
3513 case MVT::v8f16: return fastEmit_ISD_FRINT_MVT_v8f16_r(RetVT, Op0);
3514 case MVT::v2f32: return fastEmit_ISD_FRINT_MVT_v2f32_r(RetVT, Op0);
3515 case MVT::v4f32: return fastEmit_ISD_FRINT_MVT_v4f32_r(RetVT, Op0);
3516 case MVT::v2f64: return fastEmit_ISD_FRINT_MVT_v2f64_r(RetVT, Op0);
3517 default: return Register();
3518 }
3519}
3520
3521// FastEmit functions for ISD::FROUND.
3522
3523Register fastEmit_ISD_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
3524 if (RetVT.SimpleTy != MVT::f16)
3525 return Register();
3526 if ((Subtarget->hasFullFP16())) {
3527 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAHr, RC: &AArch64::FPR16RegClass, Op0);
3528 }
3529 return Register();
3530}
3531
3532Register fastEmit_ISD_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3533 if (RetVT.SimpleTy != MVT::f32)
3534 return Register();
3535 if ((Subtarget->hasFPARMv8())) {
3536 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTASr, RC: &AArch64::FPR32RegClass, Op0);
3537 }
3538 return Register();
3539}
3540
3541Register fastEmit_ISD_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3542 if (RetVT.SimpleTy != MVT::f64)
3543 return Register();
3544 if ((Subtarget->hasFPARMv8())) {
3545 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTADr, RC: &AArch64::FPR64RegClass, Op0);
3546 }
3547 return Register();
3548}
3549
3550Register fastEmit_ISD_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3551 if (RetVT.SimpleTy != MVT::v4f16)
3552 return Register();
3553 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3554 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f16, RC: &AArch64::FPR64RegClass, Op0);
3555 }
3556 return Register();
3557}
3558
3559Register fastEmit_ISD_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
3560 if (RetVT.SimpleTy != MVT::v8f16)
3561 return Register();
3562 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3563 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv8f16, RC: &AArch64::FPR128RegClass, Op0);
3564 }
3565 return Register();
3566}
3567
3568Register fastEmit_ISD_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3569 if (RetVT.SimpleTy != MVT::v2f32)
3570 return Register();
3571 if ((Subtarget->isNeonAvailable())) {
3572 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f32, RC: &AArch64::FPR64RegClass, Op0);
3573 }
3574 return Register();
3575}
3576
3577Register fastEmit_ISD_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3578 if (RetVT.SimpleTy != MVT::v4f32)
3579 return Register();
3580 if ((Subtarget->isNeonAvailable())) {
3581 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f32, RC: &AArch64::FPR128RegClass, Op0);
3582 }
3583 return Register();
3584}
3585
3586Register fastEmit_ISD_FROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
3587 if (RetVT.SimpleTy != MVT::v2f64)
3588 return Register();
3589 if ((Subtarget->isNeonAvailable())) {
3590 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f64, RC: &AArch64::FPR128RegClass, Op0);
3591 }
3592 return Register();
3593}
3594
3595Register fastEmit_ISD_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
3596 switch (VT.SimpleTy) {
3597 case MVT::f16: return fastEmit_ISD_FROUND_MVT_f16_r(RetVT, Op0);
3598 case MVT::f32: return fastEmit_ISD_FROUND_MVT_f32_r(RetVT, Op0);
3599 case MVT::f64: return fastEmit_ISD_FROUND_MVT_f64_r(RetVT, Op0);
3600 case MVT::v4f16: return fastEmit_ISD_FROUND_MVT_v4f16_r(RetVT, Op0);
3601 case MVT::v8f16: return fastEmit_ISD_FROUND_MVT_v8f16_r(RetVT, Op0);
3602 case MVT::v2f32: return fastEmit_ISD_FROUND_MVT_v2f32_r(RetVT, Op0);
3603 case MVT::v4f32: return fastEmit_ISD_FROUND_MVT_v4f32_r(RetVT, Op0);
3604 case MVT::v2f64: return fastEmit_ISD_FROUND_MVT_v2f64_r(RetVT, Op0);
3605 default: return Register();
3606 }
3607}
3608
3609// FastEmit functions for ISD::FROUNDEVEN.
3610
3611Register fastEmit_ISD_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
3612 if (RetVT.SimpleTy != MVT::f16)
3613 return Register();
3614 if ((Subtarget->hasFullFP16())) {
3615 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNHr, RC: &AArch64::FPR16RegClass, Op0);
3616 }
3617 return Register();
3618}
3619
3620Register fastEmit_ISD_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
3621 if (RetVT.SimpleTy != MVT::f32)
3622 return Register();
3623 if ((Subtarget->hasFPARMv8())) {
3624 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNSr, RC: &AArch64::FPR32RegClass, Op0);
3625 }
3626 return Register();
3627}
3628
3629Register fastEmit_ISD_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
3630 if (RetVT.SimpleTy != MVT::f64)
3631 return Register();
3632 if ((Subtarget->hasFPARMv8())) {
3633 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNDr, RC: &AArch64::FPR64RegClass, Op0);
3634 }
3635 return Register();
3636}
3637
3638Register fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
3639 if (RetVT.SimpleTy != MVT::v4f16)
3640 return Register();
3641 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3642 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f16, RC: &AArch64::FPR64RegClass, Op0);
3643 }
3644 return Register();
3645}
3646
3647Register fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
3648 if (RetVT.SimpleTy != MVT::v8f16)
3649 return Register();
3650 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3651 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv8f16, RC: &AArch64::FPR128RegClass, Op0);
3652 }
3653 return Register();
3654}
3655
3656Register fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
3657 if (RetVT.SimpleTy != MVT::v2f32)
3658 return Register();
3659 if ((Subtarget->isNeonAvailable())) {
3660 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f32, RC: &AArch64::FPR64RegClass, Op0);
3661 }
3662 return Register();
3663}
3664
3665Register fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
3666 if (RetVT.SimpleTy != MVT::v4f32)
3667 return Register();
3668 if ((Subtarget->isNeonAvailable())) {
3669 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f32, RC: &AArch64::FPR128RegClass, Op0);
3670 }
3671 return Register();
3672}
3673
3674Register fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, Register Op0) {
3675 if (RetVT.SimpleTy != MVT::v2f64)
3676 return Register();
3677 if ((Subtarget->isNeonAvailable())) {
3678 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f64, RC: &AArch64::FPR128RegClass, Op0);
3679 }
3680 return Register();
3681}
3682
3683Register fastEmit_ISD_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
3684 switch (VT.SimpleTy) {
3685 case MVT::f16: return fastEmit_ISD_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
3686 case MVT::f32: return fastEmit_ISD_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
3687 case MVT::f64: return fastEmit_ISD_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
3688 case MVT::v4f16: return fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
3689 case MVT::v8f16: return fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
3690 case MVT::v2f32: return fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
3691 case MVT::v4f32: return fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
3692 case MVT::v2f64: return fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0);
3693 default: return Register();
3694 }
3695}
3696
3697// FastEmit functions for ISD::FSQRT.
3698
3699Register fastEmit_ISD_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
3700 if (RetVT.SimpleTy != MVT::f16)
3701 return Register();
3702 if ((Subtarget->hasFullFP16())) {
3703 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTHr, RC: &AArch64::FPR16RegClass, Op0);
3704 }
3705 return Register();
3706}
3707
3708Register fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
3709 if (RetVT.SimpleTy != MVT::f32)
3710 return Register();
3711 if ((Subtarget->hasFPARMv8())) {
3712 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTSr, RC: &AArch64::FPR32RegClass, Op0);
3713 }
3714 return Register();
3715}
3716
3717Register fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
3718 if (RetVT.SimpleTy != MVT::f64)
3719 return Register();
3720 if ((Subtarget->hasFPARMv8())) {
3721 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTDr, RC: &AArch64::FPR64RegClass, Op0);
3722 }
3723 return Register();
3724}
3725
3726Register fastEmit_ISD_FSQRT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3727 if (RetVT.SimpleTy != MVT::v4f16)
3728 return Register();
3729 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3730 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f16, RC: &AArch64::FPR64RegClass, Op0);
3731 }
3732 return Register();
3733}
3734
3735Register fastEmit_ISD_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3736 if (RetVT.SimpleTy != MVT::v8f16)
3737 return Register();
3738 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3739 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv8f16, RC: &AArch64::FPR128RegClass, Op0);
3740 }
3741 return Register();
3742}
3743
3744Register fastEmit_ISD_FSQRT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3745 if (RetVT.SimpleTy != MVT::v2f32)
3746 return Register();
3747 if ((Subtarget->isNeonAvailable())) {
3748 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f32, RC: &AArch64::FPR64RegClass, Op0);
3749 }
3750 return Register();
3751}
3752
3753Register fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3754 if (RetVT.SimpleTy != MVT::v4f32)
3755 return Register();
3756 if ((Subtarget->isNeonAvailable())) {
3757 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f32, RC: &AArch64::FPR128RegClass, Op0);
3758 }
3759 return Register();
3760}
3761
3762Register fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3763 if (RetVT.SimpleTy != MVT::v2f64)
3764 return Register();
3765 if ((Subtarget->isNeonAvailable())) {
3766 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f64, RC: &AArch64::FPR128RegClass, Op0);
3767 }
3768 return Register();
3769}
3770
3771Register fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
3772 switch (VT.SimpleTy) {
3773 case MVT::f16: return fastEmit_ISD_FSQRT_MVT_f16_r(RetVT, Op0);
3774 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
3775 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
3776 case MVT::v4f16: return fastEmit_ISD_FSQRT_MVT_v4f16_r(RetVT, Op0);
3777 case MVT::v8f16: return fastEmit_ISD_FSQRT_MVT_v8f16_r(RetVT, Op0);
3778 case MVT::v2f32: return fastEmit_ISD_FSQRT_MVT_v2f32_r(RetVT, Op0);
3779 case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0);
3780 case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0);
3781 default: return Register();
3782 }
3783}
3784
3785// FastEmit functions for ISD::FTRUNC.
3786
3787Register fastEmit_ISD_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
3788 if (RetVT.SimpleTy != MVT::f16)
3789 return Register();
3790 if ((Subtarget->hasFullFP16())) {
3791 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZHr, RC: &AArch64::FPR16RegClass, Op0);
3792 }
3793 return Register();
3794}
3795
3796Register fastEmit_ISD_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
3797 if (RetVT.SimpleTy != MVT::f32)
3798 return Register();
3799 if ((Subtarget->hasFPARMv8())) {
3800 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZSr, RC: &AArch64::FPR32RegClass, Op0);
3801 }
3802 return Register();
3803}
3804
3805Register fastEmit_ISD_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
3806 if (RetVT.SimpleTy != MVT::f64)
3807 return Register();
3808 if ((Subtarget->hasFPARMv8())) {
3809 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZDr, RC: &AArch64::FPR64RegClass, Op0);
3810 }
3811 return Register();
3812}
3813
3814Register fastEmit_ISD_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
3815 if (RetVT.SimpleTy != MVT::v4f16)
3816 return Register();
3817 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3818 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f16, RC: &AArch64::FPR64RegClass, Op0);
3819 }
3820 return Register();
3821}
3822
3823Register fastEmit_ISD_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
3824 if (RetVT.SimpleTy != MVT::v8f16)
3825 return Register();
3826 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3827 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv8f16, RC: &AArch64::FPR128RegClass, Op0);
3828 }
3829 return Register();
3830}
3831
3832Register fastEmit_ISD_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
3833 if (RetVT.SimpleTy != MVT::v2f32)
3834 return Register();
3835 if ((Subtarget->isNeonAvailable())) {
3836 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f32, RC: &AArch64::FPR64RegClass, Op0);
3837 }
3838 return Register();
3839}
3840
3841Register fastEmit_ISD_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
3842 if (RetVT.SimpleTy != MVT::v4f32)
3843 return Register();
3844 if ((Subtarget->isNeonAvailable())) {
3845 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f32, RC: &AArch64::FPR128RegClass, Op0);
3846 }
3847 return Register();
3848}
3849
3850Register fastEmit_ISD_FTRUNC_MVT_v2f64_r(MVT RetVT, Register Op0) {
3851 if (RetVT.SimpleTy != MVT::v2f64)
3852 return Register();
3853 if ((Subtarget->isNeonAvailable())) {
3854 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f64, RC: &AArch64::FPR128RegClass, Op0);
3855 }
3856 return Register();
3857}
3858
3859Register fastEmit_ISD_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
3860 switch (VT.SimpleTy) {
3861 case MVT::f16: return fastEmit_ISD_FTRUNC_MVT_f16_r(RetVT, Op0);
3862 case MVT::f32: return fastEmit_ISD_FTRUNC_MVT_f32_r(RetVT, Op0);
3863 case MVT::f64: return fastEmit_ISD_FTRUNC_MVT_f64_r(RetVT, Op0);
3864 case MVT::v4f16: return fastEmit_ISD_FTRUNC_MVT_v4f16_r(RetVT, Op0);
3865 case MVT::v8f16: return fastEmit_ISD_FTRUNC_MVT_v8f16_r(RetVT, Op0);
3866 case MVT::v2f32: return fastEmit_ISD_FTRUNC_MVT_v2f32_r(RetVT, Op0);
3867 case MVT::v4f32: return fastEmit_ISD_FTRUNC_MVT_v4f32_r(RetVT, Op0);
3868 case MVT::v2f64: return fastEmit_ISD_FTRUNC_MVT_v2f64_r(RetVT, Op0);
3869 default: return Register();
3870 }
3871}
3872
3873// FastEmit functions for ISD::LLROUND.
3874
3875Register fastEmit_ISD_LLROUND_MVT_f16_r(MVT RetVT, Register Op0) {
3876 if (RetVT.SimpleTy != MVT::i64)
3877 return Register();
3878 if ((Subtarget->hasFPRCVT())) {
3879 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDHr, RC: &AArch64::FPR64RegClass, Op0);
3880 }
3881 if ((Subtarget->hasFullFP16())) {
3882 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
3883 }
3884 return Register();
3885}
3886
3887Register fastEmit_ISD_LLROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3888 if (RetVT.SimpleTy != MVT::i64)
3889 return Register();
3890 if ((Subtarget->hasFPRCVT())) {
3891 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDSr, RC: &AArch64::FPR64RegClass, Op0);
3892 }
3893 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
3894}
3895
3896Register fastEmit_ISD_LLROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3897 if (RetVT.SimpleTy != MVT::i64)
3898 return Register();
3899 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
3900}
3901
3902Register fastEmit_ISD_LLROUND_r(MVT VT, MVT RetVT, Register Op0) {
3903 switch (VT.SimpleTy) {
3904 case MVT::f16: return fastEmit_ISD_LLROUND_MVT_f16_r(RetVT, Op0);
3905 case MVT::f32: return fastEmit_ISD_LLROUND_MVT_f32_r(RetVT, Op0);
3906 case MVT::f64: return fastEmit_ISD_LLROUND_MVT_f64_r(RetVT, Op0);
3907 default: return Register();
3908 }
3909}
3910
3911// FastEmit functions for ISD::LROUND.
3912
3913Register fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(Register Op0) {
3914 if ((Subtarget->hasFPRCVT())) {
3915 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASSHr, RC: &AArch64::FPR32RegClass, Op0);
3916 }
3917 if ((Subtarget->hasFullFP16())) {
3918 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWHr, RC: &AArch64::GPR32RegClass, Op0);
3919 }
3920 return Register();
3921}
3922
3923Register fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(Register Op0) {
3924 if ((Subtarget->hasFPRCVT())) {
3925 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDHr, RC: &AArch64::FPR64RegClass, Op0);
3926 }
3927 if ((Subtarget->hasFullFP16())) {
3928 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
3929 }
3930 return Register();
3931}
3932
3933Register fastEmit_ISD_LROUND_MVT_f16_r(MVT RetVT, Register Op0) {
3934switch (RetVT.SimpleTy) {
3935 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(Op0);
3936 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(Op0);
3937 default: return Register();
3938}
3939}
3940
3941Register fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(Register Op0) {
3942 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWSr, RC: &AArch64::GPR32RegClass, Op0);
3943}
3944
3945Register fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(Register Op0) {
3946 if ((Subtarget->hasFPRCVT())) {
3947 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDSr, RC: &AArch64::FPR64RegClass, Op0);
3948 }
3949 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
3950}
3951
3952Register fastEmit_ISD_LROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3953switch (RetVT.SimpleTy) {
3954 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(Op0);
3955 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(Op0);
3956 default: return Register();
3957}
3958}
3959
3960Register fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(Register Op0) {
3961 if ((Subtarget->hasFPRCVT())) {
3962 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASSDr, RC: &AArch64::FPR32RegClass, Op0);
3963 }
3964 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWDr, RC: &AArch64::GPR32RegClass, Op0);
3965}
3966
3967Register fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(Register Op0) {
3968 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
3969}
3970
3971Register fastEmit_ISD_LROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3972switch (RetVT.SimpleTy) {
3973 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(Op0);
3974 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(Op0);
3975 default: return Register();
3976}
3977}
3978
3979Register fastEmit_ISD_LROUND_r(MVT VT, MVT RetVT, Register Op0) {
3980 switch (VT.SimpleTy) {
3981 case MVT::f16: return fastEmit_ISD_LROUND_MVT_f16_r(RetVT, Op0);
3982 case MVT::f32: return fastEmit_ISD_LROUND_MVT_f32_r(RetVT, Op0);
3983 case MVT::f64: return fastEmit_ISD_LROUND_MVT_f64_r(RetVT, Op0);
3984 default: return Register();
3985 }
3986}
3987
3988// FastEmit functions for ISD::SINT_TO_FP.
3989
3990Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
3991 if ((Subtarget->hasFPRCVT())) {
3992 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
3993 }
3994 if ((Subtarget->hasFullFP16())) {
3995 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
3996 }
3997 return Register();
3998}
3999
4000Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
4001 if ((Subtarget->hasFPARMv8())) {
4002 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
4003 }
4004 return Register();
4005}
4006
4007Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
4008 if ((Subtarget->hasFPRCVT())) {
4009 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
4010 }
4011 if ((Subtarget->hasFPARMv8())) {
4012 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
4013 }
4014 return Register();
4015}
4016
4017Register fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
4018switch (RetVT.SimpleTy) {
4019 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
4020 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
4021 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
4022 default: return Register();
4023}
4024}
4025
4026Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
4027 if ((Subtarget->hasFPRCVT())) {
4028 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
4029 }
4030 if ((Subtarget->hasFullFP16())) {
4031 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
4032 }
4033 return Register();
4034}
4035
4036Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
4037 if ((Subtarget->hasFPRCVT())) {
4038 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
4039 }
4040 if ((Subtarget->hasFPARMv8())) {
4041 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
4042 }
4043 return Register();
4044}
4045
4046Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
4047 if ((Subtarget->hasFPARMv8())) {
4048 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
4049 }
4050 return Register();
4051}
4052
4053Register fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
4054switch (RetVT.SimpleTy) {
4055 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
4056 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
4057 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
4058 default: return Register();
4059}
4060}
4061
4062Register fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
4063 if (RetVT.SimpleTy != MVT::v4f16)
4064 return Register();
4065 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4066 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
4067 }
4068 return Register();
4069}
4070
4071Register fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
4072 if (RetVT.SimpleTy != MVT::v8f16)
4073 return Register();
4074 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4075 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
4076 }
4077 return Register();
4078}
4079
4080Register fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
4081 if (RetVT.SimpleTy != MVT::v2f32)
4082 return Register();
4083 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4084 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
4085 }
4086 return Register();
4087}
4088
4089Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
4090 if (RetVT.SimpleTy != MVT::v4f32)
4091 return Register();
4092 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4093 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
4094 }
4095 return Register();
4096}
4097
4098Register fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
4099 if (RetVT.SimpleTy != MVT::v2f64)
4100 return Register();
4101 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4102 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
4103 }
4104 return Register();
4105}
4106
4107Register fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
4108 switch (VT.SimpleTy) {
4109 case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
4110 case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
4111 case MVT::v4i16: return fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
4112 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
4113 case MVT::v2i32: return fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
4114 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
4115 case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
4116 default: return Register();
4117 }
4118}
4119
4120// FastEmit functions for ISD::SPLAT_VECTOR.
4121
4122Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(Register Op0) {
4123 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4124 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_B, RC: &AArch64::ZPRRegClass, Op0);
4125 }
4126 return Register();
4127}
4128
4129Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(Register Op0) {
4130 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4131 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_H, RC: &AArch64::ZPRRegClass, Op0);
4132 }
4133 return Register();
4134}
4135
4136Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(Register Op0) {
4137 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4138 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_S, RC: &AArch64::ZPRRegClass, Op0);
4139 }
4140 return Register();
4141}
4142
4143Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(MVT RetVT, Register Op0) {
4144switch (RetVT.SimpleTy) {
4145 case MVT::nxv16i8: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(Op0);
4146 case MVT::nxv8i16: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(Op0);
4147 case MVT::nxv4i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(Op0);
4148 default: return Register();
4149}
4150}
4151
4152Register fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(MVT RetVT, Register Op0) {
4153 if (RetVT.SimpleTy != MVT::nxv2i64)
4154 return Register();
4155 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4156 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_D, RC: &AArch64::ZPRRegClass, Op0);
4157 }
4158 return Register();
4159}
4160
4161Register fastEmit_ISD_SPLAT_VECTOR_r(MVT VT, MVT RetVT, Register Op0) {
4162 switch (VT.SimpleTy) {
4163 case MVT::i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(RetVT, Op0);
4164 case MVT::i64: return fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(RetVT, Op0);
4165 default: return Register();
4166 }
4167}
4168
4169// FastEmit functions for ISD::STRICT_FCEIL.
4170
4171Register fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
4172 if (RetVT.SimpleTy != MVT::f16)
4173 return Register();
4174 if ((Subtarget->hasFullFP16())) {
4175 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPHr, RC: &AArch64::FPR16RegClass, Op0);
4176 }
4177 return Register();
4178}
4179
4180Register fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
4181 if (RetVT.SimpleTy != MVT::f32)
4182 return Register();
4183 if ((Subtarget->hasFPARMv8())) {
4184 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPSr, RC: &AArch64::FPR32RegClass, Op0);
4185 }
4186 return Register();
4187}
4188
4189Register fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
4190 if (RetVT.SimpleTy != MVT::f64)
4191 return Register();
4192 if ((Subtarget->hasFPARMv8())) {
4193 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPDr, RC: &AArch64::FPR64RegClass, Op0);
4194 }
4195 return Register();
4196}
4197
4198Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
4199 if (RetVT.SimpleTy != MVT::v4f16)
4200 return Register();
4201 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4202 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f16, RC: &AArch64::FPR64RegClass, Op0);
4203 }
4204 return Register();
4205}
4206
4207Register fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
4208 if (RetVT.SimpleTy != MVT::v8f16)
4209 return Register();
4210 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4211 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv8f16, RC: &AArch64::FPR128RegClass, Op0);
4212 }
4213 return Register();
4214}
4215
4216Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
4217 if (RetVT.SimpleTy != MVT::v2f32)
4218 return Register();
4219 if ((Subtarget->isNeonAvailable())) {
4220 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f32, RC: &AArch64::FPR64RegClass, Op0);
4221 }
4222 return Register();
4223}
4224
4225Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
4226 if (RetVT.SimpleTy != MVT::v4f32)
4227 return Register();
4228 if ((Subtarget->isNeonAvailable())) {
4229 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f32, RC: &AArch64::FPR128RegClass, Op0);
4230 }
4231 return Register();
4232}
4233
4234Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(MVT RetVT, Register Op0) {
4235 if (RetVT.SimpleTy != MVT::v2f64)
4236 return Register();
4237 if ((Subtarget->isNeonAvailable())) {
4238 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f64, RC: &AArch64::FPR128RegClass, Op0);
4239 }
4240 return Register();
4241}
4242
4243Register fastEmit_ISD_STRICT_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
4244 switch (VT.SimpleTy) {
4245 case MVT::f16: return fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(RetVT, Op0);
4246 case MVT::f32: return fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(RetVT, Op0);
4247 case MVT::f64: return fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(RetVT, Op0);
4248 case MVT::v4f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(RetVT, Op0);
4249 case MVT::v8f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(RetVT, Op0);
4250 case MVT::v2f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(RetVT, Op0);
4251 case MVT::v4f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(RetVT, Op0);
4252 case MVT::v2f64: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(RetVT, Op0);
4253 default: return Register();
4254 }
4255}
4256
4257// FastEmit functions for ISD::STRICT_FFLOOR.
4258
4259Register fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
4260 if (RetVT.SimpleTy != MVT::f16)
4261 return Register();
4262 if ((Subtarget->hasFullFP16())) {
4263 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMHr, RC: &AArch64::FPR16RegClass, Op0);
4264 }
4265 return Register();
4266}
4267
4268Register fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
4269 if (RetVT.SimpleTy != MVT::f32)
4270 return Register();
4271 if ((Subtarget->hasFPARMv8())) {
4272 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMSr, RC: &AArch64::FPR32RegClass, Op0);
4273 }
4274 return Register();
4275}
4276
4277Register fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
4278 if (RetVT.SimpleTy != MVT::f64)
4279 return Register();
4280 if ((Subtarget->hasFPARMv8())) {
4281 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMDr, RC: &AArch64::FPR64RegClass, Op0);
4282 }
4283 return Register();
4284}
4285
4286Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
4287 if (RetVT.SimpleTy != MVT::v4f16)
4288 return Register();
4289 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4290 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f16, RC: &AArch64::FPR64RegClass, Op0);
4291 }
4292 return Register();
4293}
4294
4295Register fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
4296 if (RetVT.SimpleTy != MVT::v8f16)
4297 return Register();
4298 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4299 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv8f16, RC: &AArch64::FPR128RegClass, Op0);
4300 }
4301 return Register();
4302}
4303
4304Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
4305 if (RetVT.SimpleTy != MVT::v2f32)
4306 return Register();
4307 if ((Subtarget->isNeonAvailable())) {
4308 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f32, RC: &AArch64::FPR64RegClass, Op0);
4309 }
4310 return Register();
4311}
4312
4313Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
4314 if (RetVT.SimpleTy != MVT::v4f32)
4315 return Register();
4316 if ((Subtarget->isNeonAvailable())) {
4317 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f32, RC: &AArch64::FPR128RegClass, Op0);
4318 }
4319 return Register();
4320}
4321
4322Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(MVT RetVT, Register Op0) {
4323 if (RetVT.SimpleTy != MVT::v2f64)
4324 return Register();
4325 if ((Subtarget->isNeonAvailable())) {
4326 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f64, RC: &AArch64::FPR128RegClass, Op0);
4327 }
4328 return Register();
4329}
4330
4331Register fastEmit_ISD_STRICT_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
4332 switch (VT.SimpleTy) {
4333 case MVT::f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(RetVT, Op0);
4334 case MVT::f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(RetVT, Op0);
4335 case MVT::f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(RetVT, Op0);
4336 case MVT::v4f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(RetVT, Op0);
4337 case MVT::v8f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(RetVT, Op0);
4338 case MVT::v2f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(RetVT, Op0);
4339 case MVT::v4f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(RetVT, Op0);
4340 case MVT::v2f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(RetVT, Op0);
4341 default: return Register();
4342 }
4343}
4344
4345// FastEmit functions for ISD::STRICT_FNEARBYINT.
4346
4347Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
4348 if (RetVT.SimpleTy != MVT::f16)
4349 return Register();
4350 if ((Subtarget->hasFullFP16())) {
4351 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIHr, RC: &AArch64::FPR16RegClass, Op0);
4352 }
4353 return Register();
4354}
4355
4356Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
4357 if (RetVT.SimpleTy != MVT::f32)
4358 return Register();
4359 if ((Subtarget->hasFPARMv8())) {
4360 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTISr, RC: &AArch64::FPR32RegClass, Op0);
4361 }
4362 return Register();
4363}
4364
4365Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
4366 if (RetVT.SimpleTy != MVT::f64)
4367 return Register();
4368 if ((Subtarget->hasFPARMv8())) {
4369 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIDr, RC: &AArch64::FPR64RegClass, Op0);
4370 }
4371 return Register();
4372}
4373
4374Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4375 if (RetVT.SimpleTy != MVT::v4f16)
4376 return Register();
4377 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4378 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f16, RC: &AArch64::FPR64RegClass, Op0);
4379 }
4380 return Register();
4381}
4382
4383Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4384 if (RetVT.SimpleTy != MVT::v8f16)
4385 return Register();
4386 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4387 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv8f16, RC: &AArch64::FPR128RegClass, Op0);
4388 }
4389 return Register();
4390}
4391
4392Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4393 if (RetVT.SimpleTy != MVT::v2f32)
4394 return Register();
4395 if ((Subtarget->isNeonAvailable())) {
4396 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f32, RC: &AArch64::FPR64RegClass, Op0);
4397 }
4398 return Register();
4399}
4400
4401Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4402 if (RetVT.SimpleTy != MVT::v4f32)
4403 return Register();
4404 if ((Subtarget->isNeonAvailable())) {
4405 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f32, RC: &AArch64::FPR128RegClass, Op0);
4406 }
4407 return Register();
4408}
4409
4410Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4411 if (RetVT.SimpleTy != MVT::v2f64)
4412 return Register();
4413 if ((Subtarget->isNeonAvailable())) {
4414 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f64, RC: &AArch64::FPR128RegClass, Op0);
4415 }
4416 return Register();
4417}
4418
4419Register fastEmit_ISD_STRICT_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
4420 switch (VT.SimpleTy) {
4421 case MVT::f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(RetVT, Op0);
4422 case MVT::f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(RetVT, Op0);
4423 case MVT::f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(RetVT, Op0);
4424 case MVT::v4f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(RetVT, Op0);
4425 case MVT::v8f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(RetVT, Op0);
4426 case MVT::v2f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(RetVT, Op0);
4427 case MVT::v4f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(RetVT, Op0);
4428 case MVT::v2f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(RetVT, Op0);
4429 default: return Register();
4430 }
4431}
4432
4433// FastEmit functions for ISD::STRICT_FP_EXTEND.
4434
4435Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(Register Op0) {
4436 if ((Subtarget->hasFPARMv8())) {
4437 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSHr, RC: &AArch64::FPR32RegClass, Op0);
4438 }
4439 return Register();
4440}
4441
4442Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(Register Op0) {
4443 if ((Subtarget->hasFPARMv8())) {
4444 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDHr, RC: &AArch64::FPR64RegClass, Op0);
4445 }
4446 return Register();
4447}
4448
4449Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(MVT RetVT, Register Op0) {
4450switch (RetVT.SimpleTy) {
4451 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(Op0);
4452 case MVT::f64: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(Op0);
4453 default: return Register();
4454}
4455}
4456
4457Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
4458 if (RetVT.SimpleTy != MVT::f64)
4459 return Register();
4460 if ((Subtarget->hasFPARMv8())) {
4461 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDSr, RC: &AArch64::FPR64RegClass, Op0);
4462 }
4463 return Register();
4464}
4465
4466Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
4467 if (RetVT.SimpleTy != MVT::v4f32)
4468 return Register();
4469 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv4i16, RC: &AArch64::FPR128RegClass, Op0);
4470}
4471
4472Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4bf16_r(MVT RetVT, Register Op0) {
4473 if (RetVT.SimpleTy != MVT::v4f32)
4474 return Register();
4475 if ((Subtarget->isNeonAvailable())) {
4476 return fastEmitInst_r(MachineInstOpcode: AArch64::SHLLv4i16, RC: &AArch64::FPR128RegClass, Op0);
4477 }
4478 return Register();
4479}
4480
4481Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(MVT RetVT, Register Op0) {
4482 if (RetVT.SimpleTy != MVT::v2f64)
4483 return Register();
4484 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv2i32, RC: &AArch64::FPR128RegClass, Op0);
4485}
4486
4487Register fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
4488 switch (VT.SimpleTy) {
4489 case MVT::f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(RetVT, Op0);
4490 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0);
4491 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
4492 case MVT::v4bf16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4bf16_r(RetVT, Op0);
4493 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(RetVT, Op0);
4494 default: return Register();
4495 }
4496}
4497
4498// FastEmit functions for ISD::STRICT_FP_ROUND.
4499
4500Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_bf16_r(Register Op0) {
4501 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
4502 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVT, RC: &AArch64::FPR16RegClass, Op0);
4503 }
4504 return Register();
4505}
4506
4507Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_f16_r(Register Op0) {
4508 if ((Subtarget->hasFPARMv8())) {
4509 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHSr, RC: &AArch64::FPR16RegClass, Op0);
4510 }
4511 return Register();
4512}
4513
4514Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4515switch (RetVT.SimpleTy) {
4516 case MVT::bf16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_bf16_r(Op0);
4517 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_f16_r(Op0);
4518 default: return Register();
4519}
4520}
4521
4522Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(Register Op0) {
4523 if ((Subtarget->hasFPARMv8())) {
4524 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHDr, RC: &AArch64::FPR16RegClass, Op0);
4525 }
4526 return Register();
4527}
4528
4529Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(Register Op0) {
4530 if ((Subtarget->hasFPARMv8())) {
4531 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSDr, RC: &AArch64::FPR32RegClass, Op0);
4532 }
4533 return Register();
4534}
4535
4536Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4537switch (RetVT.SimpleTy) {
4538 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(Op0);
4539 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(Op0);
4540 default: return Register();
4541}
4542}
4543
4544Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Register Op0) {
4545 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
4546}
4547
4548Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Register Op0) {
4549 if ((Subtarget->hasBF16()) && (Subtarget->isNeonAvailable())) {
4550 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVTN, RC: &AArch64::FPR64RegClass, Op0);
4551 }
4552 return Register();
4553}
4554
4555Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
4556switch (RetVT.SimpleTy) {
4557 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Op0);
4558 case MVT::v4bf16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Op0);
4559 default: return Register();
4560}
4561}
4562
4563Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
4564 if (RetVT.SimpleTy != MVT::v2f32)
4565 return Register();
4566 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
4567}
4568
4569Register fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
4570 switch (VT.SimpleTy) {
4571 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(RetVT, Op0);
4572 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0);
4573 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
4574 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(RetVT, Op0);
4575 default: return Register();
4576 }
4577}
4578
4579// FastEmit functions for ISD::STRICT_FP_TO_SINT.
4580
4581Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
4582 if ((Subtarget->hasFPRCVT())) {
4583 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSHr, RC: &AArch64::FPR32RegClass, Op0);
4584 }
4585 if ((Subtarget->hasFullFP16())) {
4586 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
4587 }
4588 return Register();
4589}
4590
4591Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
4592 if ((Subtarget->hasFPRCVT())) {
4593 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDHr, RC: &AArch64::FPR64RegClass, Op0);
4594 }
4595 if ((Subtarget->hasFullFP16())) {
4596 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
4597 }
4598 return Register();
4599}
4600
4601Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
4602switch (RetVT.SimpleTy) {
4603 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
4604 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
4605 default: return Register();
4606}
4607}
4608
4609Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
4610 if ((Subtarget->hasFPARMv8())) {
4611 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
4612 }
4613 return Register();
4614}
4615
4616Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
4617 if ((Subtarget->hasFPRCVT())) {
4618 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDSr, RC: &AArch64::FPR64RegClass, Op0);
4619 }
4620 if ((Subtarget->hasFPARMv8())) {
4621 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
4622 }
4623 return Register();
4624}
4625
4626Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
4627switch (RetVT.SimpleTy) {
4628 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
4629 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
4630 default: return Register();
4631}
4632}
4633
4634Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
4635 if ((Subtarget->hasFPRCVT())) {
4636 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSDr, RC: &AArch64::FPR32RegClass, Op0);
4637 }
4638 if ((Subtarget->hasFPARMv8())) {
4639 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
4640 }
4641 return Register();
4642}
4643
4644Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
4645 if ((Subtarget->hasFPARMv8())) {
4646 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
4647 }
4648 return Register();
4649}
4650
4651Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
4652switch (RetVT.SimpleTy) {
4653 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
4654 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
4655 default: return Register();
4656}
4657}
4658
4659Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4660 if (RetVT.SimpleTy != MVT::v4i16)
4661 return Register();
4662 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4663 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
4664 }
4665 return Register();
4666}
4667
4668Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4669 if (RetVT.SimpleTy != MVT::v8i16)
4670 return Register();
4671 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4672 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
4673 }
4674 return Register();
4675}
4676
4677Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4678 if (RetVT.SimpleTy != MVT::v2i32)
4679 return Register();
4680 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4681 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
4682 }
4683 return Register();
4684}
4685
4686Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4687 if (RetVT.SimpleTy != MVT::v4i32)
4688 return Register();
4689 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4690 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
4691 }
4692 return Register();
4693}
4694
4695Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4696 if (RetVT.SimpleTy != MVT::v2i64)
4697 return Register();
4698 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4699 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
4700 }
4701 return Register();
4702}
4703
4704Register fastEmit_ISD_STRICT_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
4705 switch (VT.SimpleTy) {
4706 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
4707 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
4708 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
4709 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
4710 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
4711 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
4712 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
4713 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0);
4714 default: return Register();
4715 }
4716}
4717
4718// FastEmit functions for ISD::STRICT_FP_TO_UINT.
4719
4720Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
4721 if ((Subtarget->hasFPRCVT())) {
4722 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSHr, RC: &AArch64::FPR32RegClass, Op0);
4723 }
4724 if ((Subtarget->hasFullFP16())) {
4725 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
4726 }
4727 return Register();
4728}
4729
4730Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
4731 if ((Subtarget->hasFPRCVT())) {
4732 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDHr, RC: &AArch64::FPR64RegClass, Op0);
4733 }
4734 if ((Subtarget->hasFullFP16())) {
4735 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
4736 }
4737 return Register();
4738}
4739
4740Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
4741switch (RetVT.SimpleTy) {
4742 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
4743 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
4744 default: return Register();
4745}
4746}
4747
4748Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
4749 if ((Subtarget->hasFPARMv8())) {
4750 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
4751 }
4752 return Register();
4753}
4754
4755Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
4756 if ((Subtarget->hasFPRCVT())) {
4757 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDSr, RC: &AArch64::FPR64RegClass, Op0);
4758 }
4759 if ((Subtarget->hasFPARMv8())) {
4760 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
4761 }
4762 return Register();
4763}
4764
4765Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
4766switch (RetVT.SimpleTy) {
4767 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
4768 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
4769 default: return Register();
4770}
4771}
4772
4773Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
4774 if ((Subtarget->hasFPRCVT())) {
4775 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSDr, RC: &AArch64::FPR32RegClass, Op0);
4776 }
4777 if ((Subtarget->hasFPARMv8())) {
4778 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
4779 }
4780 return Register();
4781}
4782
4783Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
4784 if ((Subtarget->hasFPARMv8())) {
4785 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
4786 }
4787 return Register();
4788}
4789
4790Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
4791switch (RetVT.SimpleTy) {
4792 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
4793 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
4794 default: return Register();
4795}
4796}
4797
4798Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4799 if (RetVT.SimpleTy != MVT::v4i16)
4800 return Register();
4801 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4802 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
4803 }
4804 return Register();
4805}
4806
4807Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4808 if (RetVT.SimpleTy != MVT::v8i16)
4809 return Register();
4810 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4811 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
4812 }
4813 return Register();
4814}
4815
4816Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4817 if (RetVT.SimpleTy != MVT::v2i32)
4818 return Register();
4819 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4820 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
4821 }
4822 return Register();
4823}
4824
4825Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4826 if (RetVT.SimpleTy != MVT::v4i32)
4827 return Register();
4828 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4829 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
4830 }
4831 return Register();
4832}
4833
4834Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4835 if (RetVT.SimpleTy != MVT::v2i64)
4836 return Register();
4837 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4838 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
4839 }
4840 return Register();
4841}
4842
4843Register fastEmit_ISD_STRICT_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
4844 switch (VT.SimpleTy) {
4845 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
4846 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
4847 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
4848 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
4849 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
4850 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
4851 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
4852 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0);
4853 default: return Register();
4854 }
4855}
4856
4857// FastEmit functions for ISD::STRICT_FRINT.
4858
4859Register fastEmit_ISD_STRICT_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
4860 if (RetVT.SimpleTy != MVT::f16)
4861 return Register();
4862 if ((Subtarget->hasFullFP16())) {
4863 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXHr, RC: &AArch64::FPR16RegClass, Op0);
4864 }
4865 return Register();
4866}
4867
4868Register fastEmit_ISD_STRICT_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
4869 if (RetVT.SimpleTy != MVT::f32)
4870 return Register();
4871 if ((Subtarget->hasFPARMv8())) {
4872 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXSr, RC: &AArch64::FPR32RegClass, Op0);
4873 }
4874 return Register();
4875}
4876
4877Register fastEmit_ISD_STRICT_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
4878 if (RetVT.SimpleTy != MVT::f64)
4879 return Register();
4880 if ((Subtarget->hasFPARMv8())) {
4881 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXDr, RC: &AArch64::FPR64RegClass, Op0);
4882 }
4883 return Register();
4884}
4885
4886Register fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4887 if (RetVT.SimpleTy != MVT::v4f16)
4888 return Register();
4889 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4890 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f16, RC: &AArch64::FPR64RegClass, Op0);
4891 }
4892 return Register();
4893}
4894
4895Register fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4896 if (RetVT.SimpleTy != MVT::v8f16)
4897 return Register();
4898 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4899 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv8f16, RC: &AArch64::FPR128RegClass, Op0);
4900 }
4901 return Register();
4902}
4903
4904Register fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4905 if (RetVT.SimpleTy != MVT::v2f32)
4906 return Register();
4907 if ((Subtarget->isNeonAvailable())) {
4908 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f32, RC: &AArch64::FPR64RegClass, Op0);
4909 }
4910 return Register();
4911}
4912
4913Register fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4914 if (RetVT.SimpleTy != MVT::v4f32)
4915 return Register();
4916 if ((Subtarget->isNeonAvailable())) {
4917 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f32, RC: &AArch64::FPR128RegClass, Op0);
4918 }
4919 return Register();
4920}
4921
4922Register fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4923 if (RetVT.SimpleTy != MVT::v2f64)
4924 return Register();
4925 if ((Subtarget->isNeonAvailable())) {
4926 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f64, RC: &AArch64::FPR128RegClass, Op0);
4927 }
4928 return Register();
4929}
4930
4931Register fastEmit_ISD_STRICT_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
4932 switch (VT.SimpleTy) {
4933 case MVT::f16: return fastEmit_ISD_STRICT_FRINT_MVT_f16_r(RetVT, Op0);
4934 case MVT::f32: return fastEmit_ISD_STRICT_FRINT_MVT_f32_r(RetVT, Op0);
4935 case MVT::f64: return fastEmit_ISD_STRICT_FRINT_MVT_f64_r(RetVT, Op0);
4936 case MVT::v4f16: return fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(RetVT, Op0);
4937 case MVT::v8f16: return fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(RetVT, Op0);
4938 case MVT::v2f32: return fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(RetVT, Op0);
4939 case MVT::v4f32: return fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(RetVT, Op0);
4940 case MVT::v2f64: return fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(RetVT, Op0);
4941 default: return Register();
4942 }
4943}
4944
4945// FastEmit functions for ISD::STRICT_FROUND.
4946
4947Register fastEmit_ISD_STRICT_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
4948 if (RetVT.SimpleTy != MVT::f16)
4949 return Register();
4950 if ((Subtarget->hasFullFP16())) {
4951 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAHr, RC: &AArch64::FPR16RegClass, Op0);
4952 }
4953 return Register();
4954}
4955
4956Register fastEmit_ISD_STRICT_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4957 if (RetVT.SimpleTy != MVT::f32)
4958 return Register();
4959 if ((Subtarget->hasFPARMv8())) {
4960 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTASr, RC: &AArch64::FPR32RegClass, Op0);
4961 }
4962 return Register();
4963}
4964
4965Register fastEmit_ISD_STRICT_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4966 if (RetVT.SimpleTy != MVT::f64)
4967 return Register();
4968 if ((Subtarget->hasFPARMv8())) {
4969 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTADr, RC: &AArch64::FPR64RegClass, Op0);
4970 }
4971 return Register();
4972}
4973
4974Register fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
4975 if (RetVT.SimpleTy != MVT::v4f16)
4976 return Register();
4977 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4978 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f16, RC: &AArch64::FPR64RegClass, Op0);
4979 }
4980 return Register();
4981}
4982
4983Register fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
4984 if (RetVT.SimpleTy != MVT::v8f16)
4985 return Register();
4986 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4987 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv8f16, RC: &AArch64::FPR128RegClass, Op0);
4988 }
4989 return Register();
4990}
4991
4992Register fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
4993 if (RetVT.SimpleTy != MVT::v2f32)
4994 return Register();
4995 if ((Subtarget->isNeonAvailable())) {
4996 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f32, RC: &AArch64::FPR64RegClass, Op0);
4997 }
4998 return Register();
4999}
5000
5001Register fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
5002 if (RetVT.SimpleTy != MVT::v4f32)
5003 return Register();
5004 if ((Subtarget->isNeonAvailable())) {
5005 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f32, RC: &AArch64::FPR128RegClass, Op0);
5006 }
5007 return Register();
5008}
5009
5010Register fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
5011 if (RetVT.SimpleTy != MVT::v2f64)
5012 return Register();
5013 if ((Subtarget->isNeonAvailable())) {
5014 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f64, RC: &AArch64::FPR128RegClass, Op0);
5015 }
5016 return Register();
5017}
5018
5019Register fastEmit_ISD_STRICT_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
5020 switch (VT.SimpleTy) {
5021 case MVT::f16: return fastEmit_ISD_STRICT_FROUND_MVT_f16_r(RetVT, Op0);
5022 case MVT::f32: return fastEmit_ISD_STRICT_FROUND_MVT_f32_r(RetVT, Op0);
5023 case MVT::f64: return fastEmit_ISD_STRICT_FROUND_MVT_f64_r(RetVT, Op0);
5024 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(RetVT, Op0);
5025 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(RetVT, Op0);
5026 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(RetVT, Op0);
5027 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(RetVT, Op0);
5028 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(RetVT, Op0);
5029 default: return Register();
5030 }
5031}
5032
5033// FastEmit functions for ISD::STRICT_FROUNDEVEN.
5034
5035Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
5036 if (RetVT.SimpleTy != MVT::f16)
5037 return Register();
5038 if ((Subtarget->hasFullFP16())) {
5039 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNHr, RC: &AArch64::FPR16RegClass, Op0);
5040 }
5041 return Register();
5042}
5043
5044Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
5045 if (RetVT.SimpleTy != MVT::f32)
5046 return Register();
5047 if ((Subtarget->hasFPARMv8())) {
5048 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNSr, RC: &AArch64::FPR32RegClass, Op0);
5049 }
5050 return Register();
5051}
5052
5053Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
5054 if (RetVT.SimpleTy != MVT::f64)
5055 return Register();
5056 if ((Subtarget->hasFPARMv8())) {
5057 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNDr, RC: &AArch64::FPR64RegClass, Op0);
5058 }
5059 return Register();
5060}
5061
5062Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
5063 if (RetVT.SimpleTy != MVT::v4f16)
5064 return Register();
5065 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5066 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f16, RC: &AArch64::FPR64RegClass, Op0);
5067 }
5068 return Register();
5069}
5070
5071Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
5072 if (RetVT.SimpleTy != MVT::v8f16)
5073 return Register();
5074 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5075 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv8f16, RC: &AArch64::FPR128RegClass, Op0);
5076 }
5077 return Register();
5078}
5079
5080Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
5081 if (RetVT.SimpleTy != MVT::v2f32)
5082 return Register();
5083 if ((Subtarget->isNeonAvailable())) {
5084 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f32, RC: &AArch64::FPR64RegClass, Op0);
5085 }
5086 return Register();
5087}
5088
5089Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
5090 if (RetVT.SimpleTy != MVT::v4f32)
5091 return Register();
5092 if ((Subtarget->isNeonAvailable())) {
5093 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f32, RC: &AArch64::FPR128RegClass, Op0);
5094 }
5095 return Register();
5096}
5097
5098Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, Register Op0) {
5099 if (RetVT.SimpleTy != MVT::v2f64)
5100 return Register();
5101 if ((Subtarget->isNeonAvailable())) {
5102 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f64, RC: &AArch64::FPR128RegClass, Op0);
5103 }
5104 return Register();
5105}
5106
5107Register fastEmit_ISD_STRICT_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
5108 switch (VT.SimpleTy) {
5109 case MVT::f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
5110 case MVT::f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
5111 case MVT::f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
5112 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
5113 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
5114 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
5115 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
5116 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0);
5117 default: return Register();
5118 }
5119}
5120
5121// FastEmit functions for ISD::STRICT_FSQRT.
5122
5123Register fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
5124 if (RetVT.SimpleTy != MVT::f16)
5125 return Register();
5126 if ((Subtarget->hasFullFP16())) {
5127 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTHr, RC: &AArch64::FPR16RegClass, Op0);
5128 }
5129 return Register();
5130}
5131
5132Register fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
5133 if (RetVT.SimpleTy != MVT::f32)
5134 return Register();
5135 if ((Subtarget->hasFPARMv8())) {
5136 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTSr, RC: &AArch64::FPR32RegClass, Op0);
5137 }
5138 return Register();
5139}
5140
5141Register fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
5142 if (RetVT.SimpleTy != MVT::f64)
5143 return Register();
5144 if ((Subtarget->hasFPARMv8())) {
5145 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTDr, RC: &AArch64::FPR64RegClass, Op0);
5146 }
5147 return Register();
5148}
5149
5150Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(MVT RetVT, Register Op0) {
5151 if (RetVT.SimpleTy != MVT::v4f16)
5152 return Register();
5153 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5154 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f16, RC: &AArch64::FPR64RegClass, Op0);
5155 }
5156 return Register();
5157}
5158
5159Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
5160 if (RetVT.SimpleTy != MVT::v8f16)
5161 return Register();
5162 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5163 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv8f16, RC: &AArch64::FPR128RegClass, Op0);
5164 }
5165 return Register();
5166}
5167
5168Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(MVT RetVT, Register Op0) {
5169 if (RetVT.SimpleTy != MVT::v2f32)
5170 return Register();
5171 if ((Subtarget->isNeonAvailable())) {
5172 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f32, RC: &AArch64::FPR64RegClass, Op0);
5173 }
5174 return Register();
5175}
5176
5177Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
5178 if (RetVT.SimpleTy != MVT::v4f32)
5179 return Register();
5180 if ((Subtarget->isNeonAvailable())) {
5181 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f32, RC: &AArch64::FPR128RegClass, Op0);
5182 }
5183 return Register();
5184}
5185
5186Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
5187 if (RetVT.SimpleTy != MVT::v2f64)
5188 return Register();
5189 if ((Subtarget->isNeonAvailable())) {
5190 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f64, RC: &AArch64::FPR128RegClass, Op0);
5191 }
5192 return Register();
5193}
5194
5195Register fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
5196 switch (VT.SimpleTy) {
5197 case MVT::f16: return fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(RetVT, Op0);
5198 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0);
5199 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0);
5200 case MVT::v4f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(RetVT, Op0);
5201 case MVT::v8f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(RetVT, Op0);
5202 case MVT::v2f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(RetVT, Op0);
5203 case MVT::v4f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(RetVT, Op0);
5204 case MVT::v2f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(RetVT, Op0);
5205 default: return Register();
5206 }
5207}
5208
5209// FastEmit functions for ISD::STRICT_FTRUNC.
5210
5211Register fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
5212 if (RetVT.SimpleTy != MVT::f16)
5213 return Register();
5214 if ((Subtarget->hasFullFP16())) {
5215 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZHr, RC: &AArch64::FPR16RegClass, Op0);
5216 }
5217 return Register();
5218}
5219
5220Register fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
5221 if (RetVT.SimpleTy != MVT::f32)
5222 return Register();
5223 if ((Subtarget->hasFPARMv8())) {
5224 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZSr, RC: &AArch64::FPR32RegClass, Op0);
5225 }
5226 return Register();
5227}
5228
5229Register fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
5230 if (RetVT.SimpleTy != MVT::f64)
5231 return Register();
5232 if ((Subtarget->hasFPARMv8())) {
5233 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZDr, RC: &AArch64::FPR64RegClass, Op0);
5234 }
5235 return Register();
5236}
5237
5238Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
5239 if (RetVT.SimpleTy != MVT::v4f16)
5240 return Register();
5241 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5242 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f16, RC: &AArch64::FPR64RegClass, Op0);
5243 }
5244 return Register();
5245}
5246
5247Register fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
5248 if (RetVT.SimpleTy != MVT::v8f16)
5249 return Register();
5250 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5251 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv8f16, RC: &AArch64::FPR128RegClass, Op0);
5252 }
5253 return Register();
5254}
5255
5256Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
5257 if (RetVT.SimpleTy != MVT::v2f32)
5258 return Register();
5259 if ((Subtarget->isNeonAvailable())) {
5260 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f32, RC: &AArch64::FPR64RegClass, Op0);
5261 }
5262 return Register();
5263}
5264
5265Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
5266 if (RetVT.SimpleTy != MVT::v4f32)
5267 return Register();
5268 if ((Subtarget->isNeonAvailable())) {
5269 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f32, RC: &AArch64::FPR128RegClass, Op0);
5270 }
5271 return Register();
5272}
5273
5274Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(MVT RetVT, Register Op0) {
5275 if (RetVT.SimpleTy != MVT::v2f64)
5276 return Register();
5277 if ((Subtarget->isNeonAvailable())) {
5278 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f64, RC: &AArch64::FPR128RegClass, Op0);
5279 }
5280 return Register();
5281}
5282
5283Register fastEmit_ISD_STRICT_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
5284 switch (VT.SimpleTy) {
5285 case MVT::f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(RetVT, Op0);
5286 case MVT::f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(RetVT, Op0);
5287 case MVT::f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(RetVT, Op0);
5288 case MVT::v4f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(RetVT, Op0);
5289 case MVT::v8f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(RetVT, Op0);
5290 case MVT::v2f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(RetVT, Op0);
5291 case MVT::v4f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(RetVT, Op0);
5292 case MVT::v2f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(RetVT, Op0);
5293 default: return Register();
5294 }
5295}
5296
5297// FastEmit functions for ISD::STRICT_LLROUND.
5298
5299Register fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5300 if (RetVT.SimpleTy != MVT::i64)
5301 return Register();
5302 if ((Subtarget->hasFullFP16())) {
5303 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
5304 }
5305 return Register();
5306}
5307
5308Register fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5309 if (RetVT.SimpleTy != MVT::i64)
5310 return Register();
5311 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
5312}
5313
5314Register fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5315 if (RetVT.SimpleTy != MVT::i64)
5316 return Register();
5317 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
5318}
5319
5320Register fastEmit_ISD_STRICT_LLROUND_r(MVT VT, MVT RetVT, Register Op0) {
5321 switch (VT.SimpleTy) {
5322 case MVT::f16: return fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(RetVT, Op0);
5323 case MVT::f32: return fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(RetVT, Op0);
5324 case MVT::f64: return fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(RetVT, Op0);
5325 default: return Register();
5326 }
5327}
5328
5329// FastEmit functions for ISD::STRICT_LROUND.
5330
5331Register fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(Register Op0) {
5332 if ((Subtarget->hasFullFP16())) {
5333 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWHr, RC: &AArch64::GPR32RegClass, Op0);
5334 }
5335 return Register();
5336}
5337
5338Register fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(Register Op0) {
5339 if ((Subtarget->hasFullFP16())) {
5340 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
5341 }
5342 return Register();
5343}
5344
5345Register fastEmit_ISD_STRICT_LROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5346switch (RetVT.SimpleTy) {
5347 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(Op0);
5348 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(Op0);
5349 default: return Register();
5350}
5351}
5352
5353Register fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(Register Op0) {
5354 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWSr, RC: &AArch64::GPR32RegClass, Op0);
5355}
5356
5357Register fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(Register Op0) {
5358 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
5359}
5360
5361Register fastEmit_ISD_STRICT_LROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5362switch (RetVT.SimpleTy) {
5363 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(Op0);
5364 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(Op0);
5365 default: return Register();
5366}
5367}
5368
5369Register fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(Register Op0) {
5370 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWDr, RC: &AArch64::GPR32RegClass, Op0);
5371}
5372
5373Register fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(Register Op0) {
5374 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
5375}
5376
5377Register fastEmit_ISD_STRICT_LROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5378switch (RetVT.SimpleTy) {
5379 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(Op0);
5380 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(Op0);
5381 default: return Register();
5382}
5383}
5384
5385Register fastEmit_ISD_STRICT_LROUND_r(MVT VT, MVT RetVT, Register Op0) {
5386 switch (VT.SimpleTy) {
5387 case MVT::f16: return fastEmit_ISD_STRICT_LROUND_MVT_f16_r(RetVT, Op0);
5388 case MVT::f32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_r(RetVT, Op0);
5389 case MVT::f64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_r(RetVT, Op0);
5390 default: return Register();
5391 }
5392}
5393
5394// FastEmit functions for ISD::STRICT_SINT_TO_FP.
5395
5396Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5397 if ((Subtarget->hasFPRCVT())) {
5398 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5399 }
5400 if ((Subtarget->hasFullFP16())) {
5401 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5402 }
5403 return Register();
5404}
5405
5406Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5407 if ((Subtarget->hasFPARMv8())) {
5408 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5409 }
5410 return Register();
5411}
5412
5413Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5414 if ((Subtarget->hasFPRCVT())) {
5415 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5416 }
5417 if ((Subtarget->hasFPARMv8())) {
5418 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5419 }
5420 return Register();
5421}
5422
5423Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5424switch (RetVT.SimpleTy) {
5425 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5426 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5427 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5428 default: return Register();
5429}
5430}
5431
5432Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5433 if ((Subtarget->hasFPRCVT())) {
5434 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5435 }
5436 if ((Subtarget->hasFullFP16())) {
5437 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5438 }
5439 return Register();
5440}
5441
5442Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5443 if ((Subtarget->hasFPRCVT())) {
5444 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5445 }
5446 if ((Subtarget->hasFPARMv8())) {
5447 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5448 }
5449 return Register();
5450}
5451
5452Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5453 if ((Subtarget->hasFPARMv8())) {
5454 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5455 }
5456 return Register();
5457}
5458
5459Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5460switch (RetVT.SimpleTy) {
5461 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5462 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5463 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5464 default: return Register();
5465}
5466}
5467
5468Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5469 if (RetVT.SimpleTy != MVT::v4f16)
5470 return Register();
5471 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5472 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5473 }
5474 return Register();
5475}
5476
5477Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5478 if (RetVT.SimpleTy != MVT::v8f16)
5479 return Register();
5480 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5481 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5482 }
5483 return Register();
5484}
5485
5486Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5487 if (RetVT.SimpleTy != MVT::v2f32)
5488 return Register();
5489 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5490 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5491 }
5492 return Register();
5493}
5494
5495Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5496 if (RetVT.SimpleTy != MVT::v4f32)
5497 return Register();
5498 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5499 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5500 }
5501 return Register();
5502}
5503
5504Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5505 if (RetVT.SimpleTy != MVT::v2f64)
5506 return Register();
5507 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5508 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5509 }
5510 return Register();
5511}
5512
5513Register fastEmit_ISD_STRICT_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
5514 switch (VT.SimpleTy) {
5515 case MVT::i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
5516 case MVT::i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
5517 case MVT::v4i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
5518 case MVT::v8i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
5519 case MVT::v2i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
5520 case MVT::v4i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
5521 case MVT::v2i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
5522 default: return Register();
5523 }
5524}
5525
5526// FastEmit functions for ISD::STRICT_UINT_TO_FP.
5527
5528Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5529 if ((Subtarget->hasFPRCVT())) {
5530 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5531 }
5532 if ((Subtarget->hasFullFP16())) {
5533 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5534 }
5535 return Register();
5536}
5537
5538Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5539 if ((Subtarget->hasFPARMv8())) {
5540 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5541 }
5542 return Register();
5543}
5544
5545Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5546 if ((Subtarget->hasFPRCVT())) {
5547 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5548 }
5549 if ((Subtarget->hasFPARMv8())) {
5550 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5551 }
5552 return Register();
5553}
5554
5555Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5556switch (RetVT.SimpleTy) {
5557 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5558 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5559 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5560 default: return Register();
5561}
5562}
5563
5564Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5565 if ((Subtarget->hasFPRCVT())) {
5566 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5567 }
5568 if ((Subtarget->hasFullFP16())) {
5569 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5570 }
5571 return Register();
5572}
5573
5574Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5575 if ((Subtarget->hasFPRCVT())) {
5576 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5577 }
5578 if ((Subtarget->hasFPARMv8())) {
5579 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5580 }
5581 return Register();
5582}
5583
5584Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5585 if ((Subtarget->hasFPARMv8())) {
5586 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5587 }
5588 return Register();
5589}
5590
5591Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5592switch (RetVT.SimpleTy) {
5593 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5594 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5595 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5596 default: return Register();
5597}
5598}
5599
5600Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5601 if (RetVT.SimpleTy != MVT::v4f16)
5602 return Register();
5603 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5604 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5605 }
5606 return Register();
5607}
5608
5609Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5610 if (RetVT.SimpleTy != MVT::v8f16)
5611 return Register();
5612 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5613 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5614 }
5615 return Register();
5616}
5617
5618Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5619 if (RetVT.SimpleTy != MVT::v2f32)
5620 return Register();
5621 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5622 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5623 }
5624 return Register();
5625}
5626
5627Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5628 if (RetVT.SimpleTy != MVT::v4f32)
5629 return Register();
5630 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5631 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5632 }
5633 return Register();
5634}
5635
5636Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5637 if (RetVT.SimpleTy != MVT::v2f64)
5638 return Register();
5639 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5640 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5641 }
5642 return Register();
5643}
5644
5645Register fastEmit_ISD_STRICT_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
5646 switch (VT.SimpleTy) {
5647 case MVT::i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(RetVT, Op0);
5648 case MVT::i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(RetVT, Op0);
5649 case MVT::v4i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
5650 case MVT::v8i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
5651 case MVT::v2i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
5652 case MVT::v4i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
5653 case MVT::v2i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
5654 default: return Register();
5655 }
5656}
5657
5658// FastEmit functions for ISD::TRUNCATE.
5659
5660Register fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, Register Op0) {
5661 if (RetVT.SimpleTy != MVT::i32)
5662 return Register();
5663 return fastEmitInst_extractsubreg(RetVT, Op0, Idx: AArch64::sub_32);
5664}
5665
5666Register fastEmit_ISD_TRUNCATE_MVT_v8i16_r(MVT RetVT, Register Op0) {
5667 if (RetVT.SimpleTy != MVT::v8i8)
5668 return Register();
5669 if ((Subtarget->isNeonAvailable())) {
5670 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5671 }
5672 return Register();
5673}
5674
5675Register fastEmit_ISD_TRUNCATE_MVT_v4i32_r(MVT RetVT, Register Op0) {
5676 if (RetVT.SimpleTy != MVT::v4i16)
5677 return Register();
5678 if ((Subtarget->isNeonAvailable())) {
5679 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5680 }
5681 return Register();
5682}
5683
5684Register fastEmit_ISD_TRUNCATE_MVT_v2i64_r(MVT RetVT, Register Op0) {
5685 if (RetVT.SimpleTy != MVT::v2i32)
5686 return Register();
5687 if ((Subtarget->isNeonAvailable())) {
5688 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5689 }
5690 return Register();
5691}
5692
5693Register fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, Register Op0) {
5694 switch (VT.SimpleTy) {
5695 case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0);
5696 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i16_r(RetVT, Op0);
5697 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_MVT_v4i32_r(RetVT, Op0);
5698 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_MVT_v2i64_r(RetVT, Op0);
5699 default: return Register();
5700 }
5701}
5702
5703// FastEmit functions for ISD::TRUNCATE_SSAT_S.
5704
5705Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v8i16_r(MVT RetVT, Register Op0) {
5706 if (RetVT.SimpleTy != MVT::v8i8)
5707 return Register();
5708 if ((Subtarget->isNeonAvailable())) {
5709 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5710 }
5711 return Register();
5712}
5713
5714Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v4i32_r(MVT RetVT, Register Op0) {
5715 if (RetVT.SimpleTy != MVT::v4i16)
5716 return Register();
5717 if ((Subtarget->isNeonAvailable())) {
5718 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5719 }
5720 return Register();
5721}
5722
5723Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v2i64_r(MVT RetVT, Register Op0) {
5724 if (RetVT.SimpleTy != MVT::v2i32)
5725 return Register();
5726 if ((Subtarget->isNeonAvailable())) {
5727 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5728 }
5729 return Register();
5730}
5731
5732Register fastEmit_ISD_TRUNCATE_SSAT_S_r(MVT VT, MVT RetVT, Register Op0) {
5733 switch (VT.SimpleTy) {
5734 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v8i16_r(RetVT, Op0);
5735 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v4i32_r(RetVT, Op0);
5736 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v2i64_r(RetVT, Op0);
5737 default: return Register();
5738 }
5739}
5740
5741// FastEmit functions for ISD::TRUNCATE_SSAT_U.
5742
5743Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v8i16_r(MVT RetVT, Register Op0) {
5744 if (RetVT.SimpleTy != MVT::v8i8)
5745 return Register();
5746 if ((Subtarget->isNeonAvailable())) {
5747 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5748 }
5749 return Register();
5750}
5751
5752Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v4i32_r(MVT RetVT, Register Op0) {
5753 if (RetVT.SimpleTy != MVT::v4i16)
5754 return Register();
5755 if ((Subtarget->isNeonAvailable())) {
5756 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5757 }
5758 return Register();
5759}
5760
5761Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v2i64_r(MVT RetVT, Register Op0) {
5762 if (RetVT.SimpleTy != MVT::v2i32)
5763 return Register();
5764 if ((Subtarget->isNeonAvailable())) {
5765 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5766 }
5767 return Register();
5768}
5769
5770Register fastEmit_ISD_TRUNCATE_SSAT_U_r(MVT VT, MVT RetVT, Register Op0) {
5771 switch (VT.SimpleTy) {
5772 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v8i16_r(RetVT, Op0);
5773 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v4i32_r(RetVT, Op0);
5774 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v2i64_r(RetVT, Op0);
5775 default: return Register();
5776 }
5777}
5778
5779// FastEmit functions for ISD::TRUNCATE_USAT_U.
5780
5781Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v8i16_r(MVT RetVT, Register Op0) {
5782 if (RetVT.SimpleTy != MVT::v8i8)
5783 return Register();
5784 if ((Subtarget->isNeonAvailable())) {
5785 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5786 }
5787 return Register();
5788}
5789
5790Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v4i32_r(MVT RetVT, Register Op0) {
5791 if (RetVT.SimpleTy != MVT::v4i16)
5792 return Register();
5793 if ((Subtarget->isNeonAvailable())) {
5794 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5795 }
5796 return Register();
5797}
5798
5799Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v2i64_r(MVT RetVT, Register Op0) {
5800 if (RetVT.SimpleTy != MVT::v2i32)
5801 return Register();
5802 if ((Subtarget->isNeonAvailable())) {
5803 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5804 }
5805 return Register();
5806}
5807
5808Register fastEmit_ISD_TRUNCATE_USAT_U_r(MVT VT, MVT RetVT, Register Op0) {
5809 switch (VT.SimpleTy) {
5810 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v8i16_r(RetVT, Op0);
5811 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v4i32_r(RetVT, Op0);
5812 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v2i64_r(RetVT, Op0);
5813 default: return Register();
5814 }
5815}
5816
5817// FastEmit functions for ISD::UINT_TO_FP.
5818
5819Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5820 if ((Subtarget->hasFPRCVT())) {
5821 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5822 }
5823 if ((Subtarget->hasFullFP16())) {
5824 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5825 }
5826 return Register();
5827}
5828
5829Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5830 if ((Subtarget->hasFPARMv8())) {
5831 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5832 }
5833 return Register();
5834}
5835
5836Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5837 if ((Subtarget->hasFPRCVT())) {
5838 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5839 }
5840 if ((Subtarget->hasFPARMv8())) {
5841 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5842 }
5843 return Register();
5844}
5845
5846Register fastEmit_ISD_UINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5847switch (RetVT.SimpleTy) {
5848 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5849 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5850 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5851 default: return Register();
5852}
5853}
5854
5855Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5856 if ((Subtarget->hasFPRCVT())) {
5857 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5858 }
5859 if ((Subtarget->hasFullFP16())) {
5860 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5861 }
5862 return Register();
5863}
5864
5865Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5866 if ((Subtarget->hasFPRCVT())) {
5867 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5868 }
5869 if ((Subtarget->hasFPARMv8())) {
5870 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5871 }
5872 return Register();
5873}
5874
5875Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5876 if ((Subtarget->hasFPARMv8())) {
5877 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5878 }
5879 return Register();
5880}
5881
5882Register fastEmit_ISD_UINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5883switch (RetVT.SimpleTy) {
5884 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5885 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5886 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5887 default: return Register();
5888}
5889}
5890
5891Register fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5892 if (RetVT.SimpleTy != MVT::v4f16)
5893 return Register();
5894 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5895 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5896 }
5897 return Register();
5898}
5899
5900Register fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5901 if (RetVT.SimpleTy != MVT::v8f16)
5902 return Register();
5903 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5904 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5905 }
5906 return Register();
5907}
5908
5909Register fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5910 if (RetVT.SimpleTy != MVT::v2f32)
5911 return Register();
5912 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5913 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5914 }
5915 return Register();
5916}
5917
5918Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5919 if (RetVT.SimpleTy != MVT::v4f32)
5920 return Register();
5921 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5922 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5923 }
5924 return Register();
5925}
5926
5927Register fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5928 if (RetVT.SimpleTy != MVT::v2f64)
5929 return Register();
5930 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5931 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5932 }
5933 return Register();
5934}
5935
5936Register fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
5937 switch (VT.SimpleTy) {
5938 case MVT::i32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_r(RetVT, Op0);
5939 case MVT::i64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_r(RetVT, Op0);
5940 case MVT::v4i16: return fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
5941 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
5942 case MVT::v2i32: return fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
5943 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
5944 case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
5945 default: return Register();
5946 }
5947}
5948
5949// FastEmit functions for ISD::VECREDUCE_ADD.
5950
5951Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i8_r(MVT RetVT, Register Op0) {
5952 if (RetVT.SimpleTy != MVT::i8)
5953 return Register();
5954 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
5955}
5956
5957Register fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(MVT RetVT, Register Op0) {
5958 if (RetVT.SimpleTy != MVT::i8)
5959 return Register();
5960 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
5961}
5962
5963Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i16_r(MVT RetVT, Register Op0) {
5964 if (RetVT.SimpleTy != MVT::i16)
5965 return Register();
5966 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
5967}
5968
5969Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(MVT RetVT, Register Op0) {
5970 if (RetVT.SimpleTy != MVT::i16)
5971 return Register();
5972 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
5973}
5974
5975Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(MVT RetVT, Register Op0) {
5976 if (RetVT.SimpleTy != MVT::i32)
5977 return Register();
5978 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
5979}
5980
5981Register fastEmit_ISD_VECREDUCE_ADD_MVT_v2i64_r(MVT RetVT, Register Op0) {
5982 if (RetVT.SimpleTy != MVT::i64)
5983 return Register();
5984 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
5985}
5986
5987Register fastEmit_ISD_VECREDUCE_ADD_r(MVT VT, MVT RetVT, Register Op0) {
5988 switch (VT.SimpleTy) {
5989 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i8_r(RetVT, Op0);
5990 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(RetVT, Op0);
5991 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i16_r(RetVT, Op0);
5992 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(RetVT, Op0);
5993 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(RetVT, Op0);
5994 case MVT::v2i64: return fastEmit_ISD_VECREDUCE_ADD_MVT_v2i64_r(RetVT, Op0);
5995 default: return Register();
5996 }
5997}
5998
5999// FastEmit functions for ISD::VECREDUCE_FADD.
6000
6001Register fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(MVT RetVT, Register Op0) {
6002 if (RetVT.SimpleTy != MVT::f32)
6003 return Register();
6004 return fastEmitInst_r(MachineInstOpcode: AArch64::FADDPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6005}
6006
6007Register fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(MVT RetVT, Register Op0) {
6008 if (RetVT.SimpleTy != MVT::f64)
6009 return Register();
6010 return fastEmitInst_r(MachineInstOpcode: AArch64::FADDPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6011}
6012
6013Register fastEmit_ISD_VECREDUCE_FADD_r(MVT VT, MVT RetVT, Register Op0) {
6014 switch (VT.SimpleTy) {
6015 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(RetVT, Op0);
6016 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(RetVT, Op0);
6017 default: return Register();
6018 }
6019}
6020
6021// FastEmit functions for ISD::VECREDUCE_FMAX.
6022
6023Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f16_r(MVT RetVT, Register Op0) {
6024 if (RetVT.SimpleTy != MVT::f16)
6025 return Register();
6026 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6027 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6028 }
6029 return Register();
6030}
6031
6032Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v8f16_r(MVT RetVT, Register Op0) {
6033 if (RetVT.SimpleTy != MVT::f16)
6034 return Register();
6035 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6036 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6037 }
6038 return Register();
6039}
6040
6041Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f32_r(MVT RetVT, Register Op0) {
6042 if (RetVT.SimpleTy != MVT::f32)
6043 return Register();
6044 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6045}
6046
6047Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f32_r(MVT RetVT, Register Op0) {
6048 if (RetVT.SimpleTy != MVT::f32)
6049 return Register();
6050 if ((Subtarget->isNeonAvailable())) {
6051 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6052 }
6053 return Register();
6054}
6055
6056Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f64_r(MVT RetVT, Register Op0) {
6057 if (RetVT.SimpleTy != MVT::f64)
6058 return Register();
6059 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6060}
6061
6062Register fastEmit_ISD_VECREDUCE_FMAX_r(MVT VT, MVT RetVT, Register Op0) {
6063 switch (VT.SimpleTy) {
6064 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f16_r(RetVT, Op0);
6065 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v8f16_r(RetVT, Op0);
6066 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f32_r(RetVT, Op0);
6067 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f32_r(RetVT, Op0);
6068 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f64_r(RetVT, Op0);
6069 default: return Register();
6070 }
6071}
6072
6073// FastEmit functions for ISD::VECREDUCE_FMAXIMUM.
6074
6075Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f16_r(MVT RetVT, Register Op0) {
6076 if (RetVT.SimpleTy != MVT::f16)
6077 return Register();
6078 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6079 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6080 }
6081 return Register();
6082}
6083
6084Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v8f16_r(MVT RetVT, Register Op0) {
6085 if (RetVT.SimpleTy != MVT::f16)
6086 return Register();
6087 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6088 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6089 }
6090 return Register();
6091}
6092
6093Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f32_r(MVT RetVT, Register Op0) {
6094 if (RetVT.SimpleTy != MVT::f32)
6095 return Register();
6096 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6097}
6098
6099Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f32_r(MVT RetVT, Register Op0) {
6100 if (RetVT.SimpleTy != MVT::f32)
6101 return Register();
6102 if ((Subtarget->isNeonAvailable())) {
6103 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6104 }
6105 return Register();
6106}
6107
6108Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f64_r(MVT RetVT, Register Op0) {
6109 if (RetVT.SimpleTy != MVT::f64)
6110 return Register();
6111 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6112}
6113
6114Register fastEmit_ISD_VECREDUCE_FMAXIMUM_r(MVT VT, MVT RetVT, Register Op0) {
6115 switch (VT.SimpleTy) {
6116 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f16_r(RetVT, Op0);
6117 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v8f16_r(RetVT, Op0);
6118 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f32_r(RetVT, Op0);
6119 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f32_r(RetVT, Op0);
6120 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f64_r(RetVT, Op0);
6121 default: return Register();
6122 }
6123}
6124
6125// FastEmit functions for ISD::VECREDUCE_FMIN.
6126
6127Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f16_r(MVT RetVT, Register Op0) {
6128 if (RetVT.SimpleTy != MVT::f16)
6129 return Register();
6130 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6131 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6132 }
6133 return Register();
6134}
6135
6136Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v8f16_r(MVT RetVT, Register Op0) {
6137 if (RetVT.SimpleTy != MVT::f16)
6138 return Register();
6139 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6140 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6141 }
6142 return Register();
6143}
6144
6145Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f32_r(MVT RetVT, Register Op0) {
6146 if (RetVT.SimpleTy != MVT::f32)
6147 return Register();
6148 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6149}
6150
6151Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f32_r(MVT RetVT, Register Op0) {
6152 if (RetVT.SimpleTy != MVT::f32)
6153 return Register();
6154 if ((Subtarget->isNeonAvailable())) {
6155 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6156 }
6157 return Register();
6158}
6159
6160Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f64_r(MVT RetVT, Register Op0) {
6161 if (RetVT.SimpleTy != MVT::f64)
6162 return Register();
6163 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6164}
6165
6166Register fastEmit_ISD_VECREDUCE_FMIN_r(MVT VT, MVT RetVT, Register Op0) {
6167 switch (VT.SimpleTy) {
6168 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f16_r(RetVT, Op0);
6169 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v8f16_r(RetVT, Op0);
6170 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f32_r(RetVT, Op0);
6171 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f32_r(RetVT, Op0);
6172 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f64_r(RetVT, Op0);
6173 default: return Register();
6174 }
6175}
6176
6177// FastEmit functions for ISD::VECREDUCE_FMINIMUM.
6178
6179Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f16_r(MVT RetVT, Register Op0) {
6180 if (RetVT.SimpleTy != MVT::f16)
6181 return Register();
6182 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6183 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6184 }
6185 return Register();
6186}
6187
6188Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v8f16_r(MVT RetVT, Register Op0) {
6189 if (RetVT.SimpleTy != MVT::f16)
6190 return Register();
6191 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6192 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6193 }
6194 return Register();
6195}
6196
6197Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f32_r(MVT RetVT, Register Op0) {
6198 if (RetVT.SimpleTy != MVT::f32)
6199 return Register();
6200 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6201}
6202
6203Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f32_r(MVT RetVT, Register Op0) {
6204 if (RetVT.SimpleTy != MVT::f32)
6205 return Register();
6206 if ((Subtarget->isNeonAvailable())) {
6207 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6208 }
6209 return Register();
6210}
6211
6212Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f64_r(MVT RetVT, Register Op0) {
6213 if (RetVT.SimpleTy != MVT::f64)
6214 return Register();
6215 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6216}
6217
6218Register fastEmit_ISD_VECREDUCE_FMINIMUM_r(MVT VT, MVT RetVT, Register Op0) {
6219 switch (VT.SimpleTy) {
6220 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f16_r(RetVT, Op0);
6221 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v8f16_r(RetVT, Op0);
6222 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f32_r(RetVT, Op0);
6223 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f32_r(RetVT, Op0);
6224 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f64_r(RetVT, Op0);
6225 default: return Register();
6226 }
6227}
6228
6229// FastEmit functions for ISD::VECREDUCE_SMAX.
6230
6231Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i8_r(MVT RetVT, Register Op0) {
6232 if (RetVT.SimpleTy != MVT::i8)
6233 return Register();
6234 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6235}
6236
6237Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v16i8_r(MVT RetVT, Register Op0) {
6238 if (RetVT.SimpleTy != MVT::i8)
6239 return Register();
6240 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6241}
6242
6243Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i16_r(MVT RetVT, Register Op0) {
6244 if (RetVT.SimpleTy != MVT::i16)
6245 return Register();
6246 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6247}
6248
6249Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i16_r(MVT RetVT, Register Op0) {
6250 if (RetVT.SimpleTy != MVT::i16)
6251 return Register();
6252 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6253}
6254
6255Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i32_r(MVT RetVT, Register Op0) {
6256 if (RetVT.SimpleTy != MVT::i32)
6257 return Register();
6258 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6259}
6260
6261Register fastEmit_ISD_VECREDUCE_SMAX_r(MVT VT, MVT RetVT, Register Op0) {
6262 switch (VT.SimpleTy) {
6263 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i8_r(RetVT, Op0);
6264 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v16i8_r(RetVT, Op0);
6265 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i16_r(RetVT, Op0);
6266 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i16_r(RetVT, Op0);
6267 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i32_r(RetVT, Op0);
6268 default: return Register();
6269 }
6270}
6271
6272// FastEmit functions for ISD::VECREDUCE_SMIN.
6273
6274Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i8_r(MVT RetVT, Register Op0) {
6275 if (RetVT.SimpleTy != MVT::i8)
6276 return Register();
6277 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6278}
6279
6280Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v16i8_r(MVT RetVT, Register Op0) {
6281 if (RetVT.SimpleTy != MVT::i8)
6282 return Register();
6283 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6284}
6285
6286Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i16_r(MVT RetVT, Register Op0) {
6287 if (RetVT.SimpleTy != MVT::i16)
6288 return Register();
6289 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6290}
6291
6292Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i16_r(MVT RetVT, Register Op0) {
6293 if (RetVT.SimpleTy != MVT::i16)
6294 return Register();
6295 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6296}
6297
6298Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i32_r(MVT RetVT, Register Op0) {
6299 if (RetVT.SimpleTy != MVT::i32)
6300 return Register();
6301 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6302}
6303
6304Register fastEmit_ISD_VECREDUCE_SMIN_r(MVT VT, MVT RetVT, Register Op0) {
6305 switch (VT.SimpleTy) {
6306 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i8_r(RetVT, Op0);
6307 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v16i8_r(RetVT, Op0);
6308 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i16_r(RetVT, Op0);
6309 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i16_r(RetVT, Op0);
6310 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i32_r(RetVT, Op0);
6311 default: return Register();
6312 }
6313}
6314
6315// FastEmit functions for ISD::VECREDUCE_UMAX.
6316
6317Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i8_r(MVT RetVT, Register Op0) {
6318 if (RetVT.SimpleTy != MVT::i8)
6319 return Register();
6320 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6321}
6322
6323Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v16i8_r(MVT RetVT, Register Op0) {
6324 if (RetVT.SimpleTy != MVT::i8)
6325 return Register();
6326 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6327}
6328
6329Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i16_r(MVT RetVT, Register Op0) {
6330 if (RetVT.SimpleTy != MVT::i16)
6331 return Register();
6332 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6333}
6334
6335Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i16_r(MVT RetVT, Register Op0) {
6336 if (RetVT.SimpleTy != MVT::i16)
6337 return Register();
6338 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6339}
6340
6341Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i32_r(MVT RetVT, Register Op0) {
6342 if (RetVT.SimpleTy != MVT::i32)
6343 return Register();
6344 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6345}
6346
6347Register fastEmit_ISD_VECREDUCE_UMAX_r(MVT VT, MVT RetVT, Register Op0) {
6348 switch (VT.SimpleTy) {
6349 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i8_r(RetVT, Op0);
6350 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v16i8_r(RetVT, Op0);
6351 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i16_r(RetVT, Op0);
6352 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i16_r(RetVT, Op0);
6353 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i32_r(RetVT, Op0);
6354 default: return Register();
6355 }
6356}
6357
6358// FastEmit functions for ISD::VECREDUCE_UMIN.
6359
6360Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i8_r(MVT RetVT, Register Op0) {
6361 if (RetVT.SimpleTy != MVT::i8)
6362 return Register();
6363 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6364}
6365
6366Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v16i8_r(MVT RetVT, Register Op0) {
6367 if (RetVT.SimpleTy != MVT::i8)
6368 return Register();
6369 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6370}
6371
6372Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i16_r(MVT RetVT, Register Op0) {
6373 if (RetVT.SimpleTy != MVT::i16)
6374 return Register();
6375 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6376}
6377
6378Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i16_r(MVT RetVT, Register Op0) {
6379 if (RetVT.SimpleTy != MVT::i16)
6380 return Register();
6381 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6382}
6383
6384Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i32_r(MVT RetVT, Register Op0) {
6385 if (RetVT.SimpleTy != MVT::i32)
6386 return Register();
6387 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6388}
6389
6390Register fastEmit_ISD_VECREDUCE_UMIN_r(MVT VT, MVT RetVT, Register Op0) {
6391 switch (VT.SimpleTy) {
6392 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i8_r(RetVT, Op0);
6393 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v16i8_r(RetVT, Op0);
6394 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i16_r(RetVT, Op0);
6395 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i16_r(RetVT, Op0);
6396 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i32_r(RetVT, Op0);
6397 default: return Register();
6398 }
6399}
6400
6401// FastEmit functions for ISD::VECTOR_REVERSE.
6402
6403Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(MVT RetVT, Register Op0) {
6404 if (RetVT.SimpleTy != MVT::nxv2i1)
6405 return Register();
6406 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6407 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_D, RC: &AArch64::PPRRegClass, Op0);
6408 }
6409 return Register();
6410}
6411
6412Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(MVT RetVT, Register Op0) {
6413 if (RetVT.SimpleTy != MVT::nxv4i1)
6414 return Register();
6415 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6416 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_S, RC: &AArch64::PPRRegClass, Op0);
6417 }
6418 return Register();
6419}
6420
6421Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(MVT RetVT, Register Op0) {
6422 if (RetVT.SimpleTy != MVT::nxv8i1)
6423 return Register();
6424 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6425 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_H, RC: &AArch64::PPRRegClass, Op0);
6426 }
6427 return Register();
6428}
6429
6430Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(MVT RetVT, Register Op0) {
6431 if (RetVT.SimpleTy != MVT::nxv16i1)
6432 return Register();
6433 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6434 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_B, RC: &AArch64::PPRRegClass, Op0);
6435 }
6436 return Register();
6437}
6438
6439Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
6440 if (RetVT.SimpleTy != MVT::nxv16i8)
6441 return Register();
6442 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6443 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_B, RC: &AArch64::ZPRRegClass, Op0);
6444 }
6445 return Register();
6446}
6447
6448Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
6449 if (RetVT.SimpleTy != MVT::nxv8i16)
6450 return Register();
6451 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6452 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6453 }
6454 return Register();
6455}
6456
6457Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
6458 if (RetVT.SimpleTy != MVT::nxv4i32)
6459 return Register();
6460 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6461 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6462 }
6463 return Register();
6464}
6465
6466Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(MVT RetVT, Register Op0) {
6467 if (RetVT.SimpleTy != MVT::nxv2i64)
6468 return Register();
6469 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6470 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6471 }
6472 return Register();
6473}
6474
6475Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(MVT RetVT, Register Op0) {
6476 if (RetVT.SimpleTy != MVT::nxv2f16)
6477 return Register();
6478 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6479 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6480 }
6481 return Register();
6482}
6483
6484Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(MVT RetVT, Register Op0) {
6485 if (RetVT.SimpleTy != MVT::nxv4f16)
6486 return Register();
6487 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6488 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6489 }
6490 return Register();
6491}
6492
6493Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
6494 if (RetVT.SimpleTy != MVT::nxv8f16)
6495 return Register();
6496 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6497 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6498 }
6499 return Register();
6500}
6501
6502Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(MVT RetVT, Register Op0) {
6503 if (RetVT.SimpleTy != MVT::nxv2bf16)
6504 return Register();
6505 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6506 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6507 }
6508 return Register();
6509}
6510
6511Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(MVT RetVT, Register Op0) {
6512 if (RetVT.SimpleTy != MVT::nxv4bf16)
6513 return Register();
6514 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6515 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6516 }
6517 return Register();
6518}
6519
6520Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(MVT RetVT, Register Op0) {
6521 if (RetVT.SimpleTy != MVT::nxv8bf16)
6522 return Register();
6523 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6524 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6525 }
6526 return Register();
6527}
6528
6529Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(MVT RetVT, Register Op0) {
6530 if (RetVT.SimpleTy != MVT::nxv2f32)
6531 return Register();
6532 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6533 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6534 }
6535 return Register();
6536}
6537
6538Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
6539 if (RetVT.SimpleTy != MVT::nxv4f32)
6540 return Register();
6541 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6542 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6543 }
6544 return Register();
6545}
6546
6547Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
6548 if (RetVT.SimpleTy != MVT::nxv2f64)
6549 return Register();
6550 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6551 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6552 }
6553 return Register();
6554}
6555
6556Register fastEmit_ISD_VECTOR_REVERSE_r(MVT VT, MVT RetVT, Register Op0) {
6557 switch (VT.SimpleTy) {
6558 case MVT::nxv2i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(RetVT, Op0);
6559 case MVT::nxv4i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(RetVT, Op0);
6560 case MVT::nxv8i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(RetVT, Op0);
6561 case MVT::nxv16i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(RetVT, Op0);
6562 case MVT::nxv16i8: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(RetVT, Op0);
6563 case MVT::nxv8i16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(RetVT, Op0);
6564 case MVT::nxv4i32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(RetVT, Op0);
6565 case MVT::nxv2i64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(RetVT, Op0);
6566 case MVT::nxv2f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(RetVT, Op0);
6567 case MVT::nxv4f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(RetVT, Op0);
6568 case MVT::nxv8f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(RetVT, Op0);
6569 case MVT::nxv2bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(RetVT, Op0);
6570 case MVT::nxv4bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(RetVT, Op0);
6571 case MVT::nxv8bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(RetVT, Op0);
6572 case MVT::nxv2f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(RetVT, Op0);
6573 case MVT::nxv4f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(RetVT, Op0);
6574 case MVT::nxv2f64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(RetVT, Op0);
6575 default: return Register();
6576 }
6577}
6578
6579// Top-level FastEmit function.
6580
6581Register fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, Register Op0) override {
6582 switch (Opcode) {
6583 case AArch64ISD::COALESCER_BARRIER: return fastEmit_AArch64ISD_COALESCER_BARRIER_r(VT, RetVT, Op0);
6584 case AArch64ISD::DUP: return fastEmit_AArch64ISD_DUP_r(VT, RetVT, Op0);
6585 case AArch64ISD::FCVTXN: return fastEmit_AArch64ISD_FCVTXN_r(VT, RetVT, Op0);
6586 case AArch64ISD::FRECPE: return fastEmit_AArch64ISD_FRECPE_r(VT, RetVT, Op0);
6587 case AArch64ISD::FRSQRTE: return fastEmit_AArch64ISD_FRSQRTE_r(VT, RetVT, Op0);
6588 case AArch64ISD::REV16: return fastEmit_AArch64ISD_REV16_r(VT, RetVT, Op0);
6589 case AArch64ISD::REV32: return fastEmit_AArch64ISD_REV32_r(VT, RetVT, Op0);
6590 case AArch64ISD::REV64: return fastEmit_AArch64ISD_REV64_r(VT, RetVT, Op0);
6591 case AArch64ISD::SADDLP: return fastEmit_AArch64ISD_SADDLP_r(VT, RetVT, Op0);
6592 case AArch64ISD::SITOF: return fastEmit_AArch64ISD_SITOF_r(VT, RetVT, Op0);
6593 case AArch64ISD::SQABS: return fastEmit_AArch64ISD_SQABS_r(VT, RetVT, Op0);
6594 case AArch64ISD::SQNEG: return fastEmit_AArch64ISD_SQNEG_r(VT, RetVT, Op0);
6595 case AArch64ISD::SUNPKHI: return fastEmit_AArch64ISD_SUNPKHI_r(VT, RetVT, Op0);
6596 case AArch64ISD::SUNPKLO: return fastEmit_AArch64ISD_SUNPKLO_r(VT, RetVT, Op0);
6597 case AArch64ISD::UADDLP: return fastEmit_AArch64ISD_UADDLP_r(VT, RetVT, Op0);
6598 case AArch64ISD::UITOF: return fastEmit_AArch64ISD_UITOF_r(VT, RetVT, Op0);
6599 case AArch64ISD::UUNPKHI: return fastEmit_AArch64ISD_UUNPKHI_r(VT, RetVT, Op0);
6600 case AArch64ISD::UUNPKLO: return fastEmit_AArch64ISD_UUNPKLO_r(VT, RetVT, Op0);
6601 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
6602 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
6603 case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0);
6604 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
6605 case ISD::CTLS: return fastEmit_ISD_CTLS_r(VT, RetVT, Op0);
6606 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
6607 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
6608 case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0);
6609 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
6610 case ISD::FCEIL: return fastEmit_ISD_FCEIL_r(VT, RetVT, Op0);
6611 case ISD::FFLOOR: return fastEmit_ISD_FFLOOR_r(VT, RetVT, Op0);
6612 case ISD::FNEARBYINT: return fastEmit_ISD_FNEARBYINT_r(VT, RetVT, Op0);
6613 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
6614 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
6615 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
6616 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
6617 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
6618 case ISD::FRINT: return fastEmit_ISD_FRINT_r(VT, RetVT, Op0);
6619 case ISD::FROUND: return fastEmit_ISD_FROUND_r(VT, RetVT, Op0);
6620 case ISD::FROUNDEVEN: return fastEmit_ISD_FROUNDEVEN_r(VT, RetVT, Op0);
6621 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
6622 case ISD::FTRUNC: return fastEmit_ISD_FTRUNC_r(VT, RetVT, Op0);
6623 case ISD::LLROUND: return fastEmit_ISD_LLROUND_r(VT, RetVT, Op0);
6624 case ISD::LROUND: return fastEmit_ISD_LROUND_r(VT, RetVT, Op0);
6625 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
6626 case ISD::SPLAT_VECTOR: return fastEmit_ISD_SPLAT_VECTOR_r(VT, RetVT, Op0);
6627 case ISD::STRICT_FCEIL: return fastEmit_ISD_STRICT_FCEIL_r(VT, RetVT, Op0);
6628 case ISD::STRICT_FFLOOR: return fastEmit_ISD_STRICT_FFLOOR_r(VT, RetVT, Op0);
6629 case ISD::STRICT_FNEARBYINT: return fastEmit_ISD_STRICT_FNEARBYINT_r(VT, RetVT, Op0);
6630 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0);
6631 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0);
6632 case ISD::STRICT_FP_TO_SINT: return fastEmit_ISD_STRICT_FP_TO_SINT_r(VT, RetVT, Op0);
6633 case ISD::STRICT_FP_TO_UINT: return fastEmit_ISD_STRICT_FP_TO_UINT_r(VT, RetVT, Op0);
6634 case ISD::STRICT_FRINT: return fastEmit_ISD_STRICT_FRINT_r(VT, RetVT, Op0);
6635 case ISD::STRICT_FROUND: return fastEmit_ISD_STRICT_FROUND_r(VT, RetVT, Op0);
6636 case ISD::STRICT_FROUNDEVEN: return fastEmit_ISD_STRICT_FROUNDEVEN_r(VT, RetVT, Op0);
6637 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0);
6638 case ISD::STRICT_FTRUNC: return fastEmit_ISD_STRICT_FTRUNC_r(VT, RetVT, Op0);
6639 case ISD::STRICT_LLROUND: return fastEmit_ISD_STRICT_LLROUND_r(VT, RetVT, Op0);
6640 case ISD::STRICT_LROUND: return fastEmit_ISD_STRICT_LROUND_r(VT, RetVT, Op0);
6641 case ISD::STRICT_SINT_TO_FP: return fastEmit_ISD_STRICT_SINT_TO_FP_r(VT, RetVT, Op0);
6642 case ISD::STRICT_UINT_TO_FP: return fastEmit_ISD_STRICT_UINT_TO_FP_r(VT, RetVT, Op0);
6643 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
6644 case ISD::TRUNCATE_SSAT_S: return fastEmit_ISD_TRUNCATE_SSAT_S_r(VT, RetVT, Op0);
6645 case ISD::TRUNCATE_SSAT_U: return fastEmit_ISD_TRUNCATE_SSAT_U_r(VT, RetVT, Op0);
6646 case ISD::TRUNCATE_USAT_U: return fastEmit_ISD_TRUNCATE_USAT_U_r(VT, RetVT, Op0);
6647 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
6648 case ISD::VECREDUCE_ADD: return fastEmit_ISD_VECREDUCE_ADD_r(VT, RetVT, Op0);
6649 case ISD::VECREDUCE_FADD: return fastEmit_ISD_VECREDUCE_FADD_r(VT, RetVT, Op0);
6650 case ISD::VECREDUCE_FMAX: return fastEmit_ISD_VECREDUCE_FMAX_r(VT, RetVT, Op0);
6651 case ISD::VECREDUCE_FMAXIMUM: return fastEmit_ISD_VECREDUCE_FMAXIMUM_r(VT, RetVT, Op0);
6652 case ISD::VECREDUCE_FMIN: return fastEmit_ISD_VECREDUCE_FMIN_r(VT, RetVT, Op0);
6653 case ISD::VECREDUCE_FMINIMUM: return fastEmit_ISD_VECREDUCE_FMINIMUM_r(VT, RetVT, Op0);
6654 case ISD::VECREDUCE_SMAX: return fastEmit_ISD_VECREDUCE_SMAX_r(VT, RetVT, Op0);
6655 case ISD::VECREDUCE_SMIN: return fastEmit_ISD_VECREDUCE_SMIN_r(VT, RetVT, Op0);
6656 case ISD::VECREDUCE_UMAX: return fastEmit_ISD_VECREDUCE_UMAX_r(VT, RetVT, Op0);
6657 case ISD::VECREDUCE_UMIN: return fastEmit_ISD_VECREDUCE_UMIN_r(VT, RetVT, Op0);
6658 case ISD::VECTOR_REVERSE: return fastEmit_ISD_VECTOR_REVERSE_r(VT, RetVT, Op0);
6659 default: return Register();
6660 }
6661}
6662
6663// FastEmit functions for AArch64ISD::ADDHN.
6664
6665Register fastEmit_AArch64ISD_ADDHN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6666 if (RetVT.SimpleTy != MVT::v8i8)
6667 return Register();
6668 if ((Subtarget->isNeonAvailable())) {
6669 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDHNv8i16_v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
6670 }
6671 return Register();
6672}
6673
6674Register fastEmit_AArch64ISD_ADDHN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6675 if (RetVT.SimpleTy != MVT::v4i16)
6676 return Register();
6677 if ((Subtarget->isNeonAvailable())) {
6678 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDHNv4i32_v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6679 }
6680 return Register();
6681}
6682
6683Register fastEmit_AArch64ISD_ADDHN_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6684 if (RetVT.SimpleTy != MVT::v2i32)
6685 return Register();
6686 if ((Subtarget->isNeonAvailable())) {
6687 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDHNv2i64_v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6688 }
6689 return Register();
6690}
6691
6692Register fastEmit_AArch64ISD_ADDHN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6693 switch (VT.SimpleTy) {
6694 case MVT::v8i16: return fastEmit_AArch64ISD_ADDHN_MVT_v8i16_rr(RetVT, Op0, Op1);
6695 case MVT::v4i32: return fastEmit_AArch64ISD_ADDHN_MVT_v4i32_rr(RetVT, Op0, Op1);
6696 case MVT::v2i64: return fastEmit_AArch64ISD_ADDHN_MVT_v2i64_rr(RetVT, Op0, Op1);
6697 default: return Register();
6698 }
6699}
6700
6701// FastEmit functions for AArch64ISD::ADDP.
6702
6703Register fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6704 if (RetVT.SimpleTy != MVT::v8i8)
6705 return Register();
6706 if ((Subtarget->isNeonAvailable())) {
6707 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
6708 }
6709 return Register();
6710}
6711
6712Register fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6713 if (RetVT.SimpleTy != MVT::v16i8)
6714 return Register();
6715 if ((Subtarget->isNeonAvailable())) {
6716 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
6717 }
6718 return Register();
6719}
6720
6721Register fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6722 if (RetVT.SimpleTy != MVT::v4i16)
6723 return Register();
6724 if ((Subtarget->isNeonAvailable())) {
6725 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6726 }
6727 return Register();
6728}
6729
6730Register fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6731 if (RetVT.SimpleTy != MVT::v8i16)
6732 return Register();
6733 if ((Subtarget->isNeonAvailable())) {
6734 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
6735 }
6736 return Register();
6737}
6738
6739Register fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6740 if (RetVT.SimpleTy != MVT::v2i32)
6741 return Register();
6742 if ((Subtarget->isNeonAvailable())) {
6743 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6744 }
6745 return Register();
6746}
6747
6748Register fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6749 if (RetVT.SimpleTy != MVT::v4i32)
6750 return Register();
6751 if ((Subtarget->isNeonAvailable())) {
6752 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
6753 }
6754 return Register();
6755}
6756
6757Register fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6758 if (RetVT.SimpleTy != MVT::v2i64)
6759 return Register();
6760 if ((Subtarget->isNeonAvailable())) {
6761 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
6762 }
6763 return Register();
6764}
6765
6766Register fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
6767 if (RetVT.SimpleTy != MVT::v4f16)
6768 return Register();
6769 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6770 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6771 }
6772 return Register();
6773}
6774
6775Register fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
6776 if (RetVT.SimpleTy != MVT::v8f16)
6777 return Register();
6778 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6779 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
6780 }
6781 return Register();
6782}
6783
6784Register fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
6785 if (RetVT.SimpleTy != MVT::v2f32)
6786 return Register();
6787 if ((Subtarget->isNeonAvailable())) {
6788 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6789 }
6790 return Register();
6791}
6792
6793Register fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
6794 if (RetVT.SimpleTy != MVT::v4f32)
6795 return Register();
6796 if ((Subtarget->isNeonAvailable())) {
6797 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
6798 }
6799 return Register();
6800}
6801
6802Register fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
6803 if (RetVT.SimpleTy != MVT::v2f64)
6804 return Register();
6805 if ((Subtarget->isNeonAvailable())) {
6806 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
6807 }
6808 return Register();
6809}
6810
6811Register fastEmit_AArch64ISD_ADDP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6812 switch (VT.SimpleTy) {
6813 case MVT::v8i8: return fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(RetVT, Op0, Op1);
6814 case MVT::v16i8: return fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(RetVT, Op0, Op1);
6815 case MVT::v4i16: return fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(RetVT, Op0, Op1);
6816 case MVT::v8i16: return fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(RetVT, Op0, Op1);
6817 case MVT::v2i32: return fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(RetVT, Op0, Op1);
6818 case MVT::v4i32: return fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(RetVT, Op0, Op1);
6819 case MVT::v2i64: return fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(RetVT, Op0, Op1);
6820 case MVT::v4f16: return fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(RetVT, Op0, Op1);
6821 case MVT::v8f16: return fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(RetVT, Op0, Op1);
6822 case MVT::v2f32: return fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(RetVT, Op0, Op1);
6823 case MVT::v4f32: return fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(RetVT, Op0, Op1);
6824 case MVT::v2f64: return fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(RetVT, Op0, Op1);
6825 default: return Register();
6826 }
6827}
6828
6829// FastEmit functions for AArch64ISD::BIC.
6830
6831Register fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6832 if (RetVT.SimpleTy != MVT::nxv16i8)
6833 return Register();
6834 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6835 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6836 }
6837 return Register();
6838}
6839
6840Register fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6841 if (RetVT.SimpleTy != MVT::nxv8i16)
6842 return Register();
6843 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6844 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6845 }
6846 return Register();
6847}
6848
6849Register fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6850 if (RetVT.SimpleTy != MVT::nxv4i32)
6851 return Register();
6852 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6853 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6854 }
6855 return Register();
6856}
6857
6858Register fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6859 if (RetVT.SimpleTy != MVT::nxv2i64)
6860 return Register();
6861 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6862 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6863 }
6864 return Register();
6865}
6866
6867Register fastEmit_AArch64ISD_BIC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6868 switch (VT.SimpleTy) {
6869 case MVT::nxv16i8: return fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(RetVT, Op0, Op1);
6870 case MVT::nxv8i16: return fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(RetVT, Op0, Op1);
6871 case MVT::nxv4i32: return fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(RetVT, Op0, Op1);
6872 case MVT::nxv2i64: return fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(RetVT, Op0, Op1);
6873 default: return Register();
6874 }
6875}
6876
6877// FastEmit functions for AArch64ISD::CMTST.
6878
6879Register fastEmit_AArch64ISD_CMTST_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6880 if (RetVT.SimpleTy != MVT::v8i8)
6881 return Register();
6882 if ((Subtarget->isNeonAvailable())) {
6883 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
6884 }
6885 return Register();
6886}
6887
6888Register fastEmit_AArch64ISD_CMTST_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6889 if (RetVT.SimpleTy != MVT::v16i8)
6890 return Register();
6891 if ((Subtarget->isNeonAvailable())) {
6892 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
6893 }
6894 return Register();
6895}
6896
6897Register fastEmit_AArch64ISD_CMTST_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6898 if (RetVT.SimpleTy != MVT::v4i16)
6899 return Register();
6900 if ((Subtarget->isNeonAvailable())) {
6901 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6902 }
6903 return Register();
6904}
6905
6906Register fastEmit_AArch64ISD_CMTST_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6907 if (RetVT.SimpleTy != MVT::v8i16)
6908 return Register();
6909 if ((Subtarget->isNeonAvailable())) {
6910 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
6911 }
6912 return Register();
6913}
6914
6915Register fastEmit_AArch64ISD_CMTST_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6916 if (RetVT.SimpleTy != MVT::v2i32)
6917 return Register();
6918 if ((Subtarget->isNeonAvailable())) {
6919 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6920 }
6921 return Register();
6922}
6923
6924Register fastEmit_AArch64ISD_CMTST_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6925 if (RetVT.SimpleTy != MVT::v4i32)
6926 return Register();
6927 if ((Subtarget->isNeonAvailable())) {
6928 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
6929 }
6930 return Register();
6931}
6932
6933Register fastEmit_AArch64ISD_CMTST_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
6934 if (RetVT.SimpleTy != MVT::v1i64)
6935 return Register();
6936 if ((Subtarget->isNeonAvailable())) {
6937 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
6938 }
6939 return Register();
6940}
6941
6942Register fastEmit_AArch64ISD_CMTST_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6943 if (RetVT.SimpleTy != MVT::v2i64)
6944 return Register();
6945 if ((Subtarget->isNeonAvailable())) {
6946 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
6947 }
6948 return Register();
6949}
6950
6951Register fastEmit_AArch64ISD_CMTST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6952 switch (VT.SimpleTy) {
6953 case MVT::v8i8: return fastEmit_AArch64ISD_CMTST_MVT_v8i8_rr(RetVT, Op0, Op1);
6954 case MVT::v16i8: return fastEmit_AArch64ISD_CMTST_MVT_v16i8_rr(RetVT, Op0, Op1);
6955 case MVT::v4i16: return fastEmit_AArch64ISD_CMTST_MVT_v4i16_rr(RetVT, Op0, Op1);
6956 case MVT::v8i16: return fastEmit_AArch64ISD_CMTST_MVT_v8i16_rr(RetVT, Op0, Op1);
6957 case MVT::v2i32: return fastEmit_AArch64ISD_CMTST_MVT_v2i32_rr(RetVT, Op0, Op1);
6958 case MVT::v4i32: return fastEmit_AArch64ISD_CMTST_MVT_v4i32_rr(RetVT, Op0, Op1);
6959 case MVT::v1i64: return fastEmit_AArch64ISD_CMTST_MVT_v1i64_rr(RetVT, Op0, Op1);
6960 case MVT::v2i64: return fastEmit_AArch64ISD_CMTST_MVT_v2i64_rr(RetVT, Op0, Op1);
6961 default: return Register();
6962 }
6963}
6964
6965// FastEmit functions for AArch64ISD::CTTZ_ELTS_ZERO_POISON.
6966
6967Register fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
6968 if (RetVT.SimpleTy != MVT::i64)
6969 return Register();
6970 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()))) {
6971 return fastEmitInst_rr(MachineInstOpcode: AArch64::FIRSTP_XPP_D, RC: &AArch64::GPR64RegClass, Op0, Op1);
6972 }
6973 return Register();
6974}
6975
6976Register fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
6977 if (RetVT.SimpleTy != MVT::i64)
6978 return Register();
6979 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()))) {
6980 return fastEmitInst_rr(MachineInstOpcode: AArch64::FIRSTP_XPP_S, RC: &AArch64::GPR64RegClass, Op0, Op1);
6981 }
6982 return Register();
6983}
6984
6985Register fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
6986 if (RetVT.SimpleTy != MVT::i64)
6987 return Register();
6988 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()))) {
6989 return fastEmitInst_rr(MachineInstOpcode: AArch64::FIRSTP_XPP_H, RC: &AArch64::GPR64RegClass, Op0, Op1);
6990 }
6991 return Register();
6992}
6993
6994Register fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
6995 if (RetVT.SimpleTy != MVT::i64)
6996 return Register();
6997 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()))) {
6998 return fastEmitInst_rr(MachineInstOpcode: AArch64::FIRSTP_XPP_B, RC: &AArch64::GPR64RegClass, Op0, Op1);
6999 }
7000 return Register();
7001}
7002
7003Register fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7004 switch (VT.SimpleTy) {
7005 case MVT::nxv2i1: return fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv2i1_rr(RetVT, Op0, Op1);
7006 case MVT::nxv4i1: return fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv4i1_rr(RetVT, Op0, Op1);
7007 case MVT::nxv8i1: return fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv8i1_rr(RetVT, Op0, Op1);
7008 case MVT::nxv16i1: return fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7009 default: return Register();
7010 }
7011}
7012
7013// FastEmit functions for AArch64ISD::FCMEQ.
7014
7015Register fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7016 if (RetVT.SimpleTy != MVT::i32)
7017 return Register();
7018 if ((Subtarget->isNeonAvailable())) {
7019 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7020 }
7021 return Register();
7022}
7023
7024Register fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7025 if (RetVT.SimpleTy != MVT::i64)
7026 return Register();
7027 if ((Subtarget->isNeonAvailable())) {
7028 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7029 }
7030 return Register();
7031}
7032
7033Register fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7034 if (RetVT.SimpleTy != MVT::v4i16)
7035 return Register();
7036 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7037 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7038 }
7039 return Register();
7040}
7041
7042Register fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7043 if (RetVT.SimpleTy != MVT::v8i16)
7044 return Register();
7045 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7046 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7047 }
7048 return Register();
7049}
7050
7051Register fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7052 if (RetVT.SimpleTy != MVT::v2i32)
7053 return Register();
7054 if ((Subtarget->isNeonAvailable())) {
7055 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7056 }
7057 return Register();
7058}
7059
7060Register fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7061 if (RetVT.SimpleTy != MVT::v4i32)
7062 return Register();
7063 if ((Subtarget->isNeonAvailable())) {
7064 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7065 }
7066 return Register();
7067}
7068
7069Register fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7070 if (RetVT.SimpleTy != MVT::v1i64)
7071 return Register();
7072 if ((Subtarget->isNeonAvailable())) {
7073 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7074 }
7075 return Register();
7076}
7077
7078Register fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7079 if (RetVT.SimpleTy != MVT::v2i64)
7080 return Register();
7081 if ((Subtarget->isNeonAvailable())) {
7082 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7083 }
7084 return Register();
7085}
7086
7087Register fastEmit_AArch64ISD_FCMEQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7088 switch (VT.SimpleTy) {
7089 case MVT::f32: return fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(RetVT, Op0, Op1);
7090 case MVT::f64: return fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(RetVT, Op0, Op1);
7091 case MVT::v4f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(RetVT, Op0, Op1);
7092 case MVT::v8f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(RetVT, Op0, Op1);
7093 case MVT::v2f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(RetVT, Op0, Op1);
7094 case MVT::v4f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(RetVT, Op0, Op1);
7095 case MVT::v1f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(RetVT, Op0, Op1);
7096 case MVT::v2f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(RetVT, Op0, Op1);
7097 default: return Register();
7098 }
7099}
7100
7101// FastEmit functions for AArch64ISD::FCMGE.
7102
7103Register fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7104 if (RetVT.SimpleTy != MVT::i32)
7105 return Register();
7106 if ((Subtarget->isNeonAvailable())) {
7107 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7108 }
7109 return Register();
7110}
7111
7112Register fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7113 if (RetVT.SimpleTy != MVT::i64)
7114 return Register();
7115 if ((Subtarget->isNeonAvailable())) {
7116 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7117 }
7118 return Register();
7119}
7120
7121Register fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7122 if (RetVT.SimpleTy != MVT::v4i16)
7123 return Register();
7124 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7125 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7126 }
7127 return Register();
7128}
7129
7130Register fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7131 if (RetVT.SimpleTy != MVT::v8i16)
7132 return Register();
7133 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7134 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7135 }
7136 return Register();
7137}
7138
7139Register fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7140 if (RetVT.SimpleTy != MVT::v2i32)
7141 return Register();
7142 if ((Subtarget->isNeonAvailable())) {
7143 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7144 }
7145 return Register();
7146}
7147
7148Register fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7149 if (RetVT.SimpleTy != MVT::v4i32)
7150 return Register();
7151 if ((Subtarget->isNeonAvailable())) {
7152 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7153 }
7154 return Register();
7155}
7156
7157Register fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7158 if (RetVT.SimpleTy != MVT::v1i64)
7159 return Register();
7160 if ((Subtarget->isNeonAvailable())) {
7161 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7162 }
7163 return Register();
7164}
7165
7166Register fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7167 if (RetVT.SimpleTy != MVT::v2i64)
7168 return Register();
7169 if ((Subtarget->isNeonAvailable())) {
7170 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7171 }
7172 return Register();
7173}
7174
7175Register fastEmit_AArch64ISD_FCMGE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7176 switch (VT.SimpleTy) {
7177 case MVT::f32: return fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(RetVT, Op0, Op1);
7178 case MVT::f64: return fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(RetVT, Op0, Op1);
7179 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(RetVT, Op0, Op1);
7180 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(RetVT, Op0, Op1);
7181 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(RetVT, Op0, Op1);
7182 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(RetVT, Op0, Op1);
7183 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(RetVT, Op0, Op1);
7184 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(RetVT, Op0, Op1);
7185 default: return Register();
7186 }
7187}
7188
7189// FastEmit functions for AArch64ISD::FCMGT.
7190
7191Register fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7192 if (RetVT.SimpleTy != MVT::i32)
7193 return Register();
7194 if ((Subtarget->isNeonAvailable())) {
7195 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7196 }
7197 return Register();
7198}
7199
7200Register fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7201 if (RetVT.SimpleTy != MVT::i64)
7202 return Register();
7203 if ((Subtarget->isNeonAvailable())) {
7204 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7205 }
7206 return Register();
7207}
7208
7209Register fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7210 if (RetVT.SimpleTy != MVT::v4i16)
7211 return Register();
7212 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7213 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7214 }
7215 return Register();
7216}
7217
7218Register fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7219 if (RetVT.SimpleTy != MVT::v8i16)
7220 return Register();
7221 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7222 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7223 }
7224 return Register();
7225}
7226
7227Register fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7228 if (RetVT.SimpleTy != MVT::v2i32)
7229 return Register();
7230 if ((Subtarget->isNeonAvailable())) {
7231 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7232 }
7233 return Register();
7234}
7235
7236Register fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7237 if (RetVT.SimpleTy != MVT::v4i32)
7238 return Register();
7239 if ((Subtarget->isNeonAvailable())) {
7240 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7241 }
7242 return Register();
7243}
7244
7245Register fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7246 if (RetVT.SimpleTy != MVT::v1i64)
7247 return Register();
7248 if ((Subtarget->isNeonAvailable())) {
7249 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7250 }
7251 return Register();
7252}
7253
7254Register fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7255 if (RetVT.SimpleTy != MVT::v2i64)
7256 return Register();
7257 if ((Subtarget->isNeonAvailable())) {
7258 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7259 }
7260 return Register();
7261}
7262
7263Register fastEmit_AArch64ISD_FCMGT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7264 switch (VT.SimpleTy) {
7265 case MVT::f32: return fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(RetVT, Op0, Op1);
7266 case MVT::f64: return fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(RetVT, Op0, Op1);
7267 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(RetVT, Op0, Op1);
7268 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(RetVT, Op0, Op1);
7269 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(RetVT, Op0, Op1);
7270 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(RetVT, Op0, Op1);
7271 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(RetVT, Op0, Op1);
7272 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(RetVT, Op0, Op1);
7273 default: return Register();
7274 }
7275}
7276
7277// FastEmit functions for AArch64ISD::FRECPS.
7278
7279Register fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7280 if (RetVT.SimpleTy != MVT::f32)
7281 return Register();
7282 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7283}
7284
7285Register fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7286 if (RetVT.SimpleTy != MVT::f64)
7287 return Register();
7288 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7289}
7290
7291Register fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7292 if (RetVT.SimpleTy != MVT::v2f32)
7293 return Register();
7294 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7295}
7296
7297Register fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7298 if (RetVT.SimpleTy != MVT::v4f32)
7299 return Register();
7300 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7301}
7302
7303Register fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7304 if (RetVT.SimpleTy != MVT::v2f64)
7305 return Register();
7306 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7307}
7308
7309Register fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7310 if (RetVT.SimpleTy != MVT::nxv8f16)
7311 return Register();
7312 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7313 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7314 }
7315 return Register();
7316}
7317
7318Register fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7319 if (RetVT.SimpleTy != MVT::nxv4f32)
7320 return Register();
7321 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7322 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7323 }
7324 return Register();
7325}
7326
7327Register fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7328 if (RetVT.SimpleTy != MVT::nxv2f64)
7329 return Register();
7330 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7331 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7332 }
7333 return Register();
7334}
7335
7336Register fastEmit_AArch64ISD_FRECPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7337 switch (VT.SimpleTy) {
7338 case MVT::f32: return fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(RetVT, Op0, Op1);
7339 case MVT::f64: return fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(RetVT, Op0, Op1);
7340 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(RetVT, Op0, Op1);
7341 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(RetVT, Op0, Op1);
7342 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(RetVT, Op0, Op1);
7343 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(RetVT, Op0, Op1);
7344 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(RetVT, Op0, Op1);
7345 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(RetVT, Op0, Op1);
7346 default: return Register();
7347 }
7348}
7349
7350// FastEmit functions for AArch64ISD::FRSQRTS.
7351
7352Register fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7353 if (RetVT.SimpleTy != MVT::f32)
7354 return Register();
7355 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7356}
7357
7358Register fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7359 if (RetVT.SimpleTy != MVT::f64)
7360 return Register();
7361 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7362}
7363
7364Register fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7365 if (RetVT.SimpleTy != MVT::v2f32)
7366 return Register();
7367 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7368}
7369
7370Register fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7371 if (RetVT.SimpleTy != MVT::v4f32)
7372 return Register();
7373 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7374}
7375
7376Register fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7377 if (RetVT.SimpleTy != MVT::v2f64)
7378 return Register();
7379 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7380}
7381
7382Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7383 if (RetVT.SimpleTy != MVT::nxv8f16)
7384 return Register();
7385 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7386 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7387 }
7388 return Register();
7389}
7390
7391Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7392 if (RetVT.SimpleTy != MVT::nxv4f32)
7393 return Register();
7394 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7395 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7396 }
7397 return Register();
7398}
7399
7400Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7401 if (RetVT.SimpleTy != MVT::nxv2f64)
7402 return Register();
7403 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7404 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7405 }
7406 return Register();
7407}
7408
7409Register fastEmit_AArch64ISD_FRSQRTS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7410 switch (VT.SimpleTy) {
7411 case MVT::f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(RetVT, Op0, Op1);
7412 case MVT::f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(RetVT, Op0, Op1);
7413 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(RetVT, Op0, Op1);
7414 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(RetVT, Op0, Op1);
7415 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(RetVT, Op0, Op1);
7416 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(RetVT, Op0, Op1);
7417 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(RetVT, Op0, Op1);
7418 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(RetVT, Op0, Op1);
7419 default: return Register();
7420 }
7421}
7422
7423// FastEmit functions for AArch64ISD::PMULL.
7424
7425Register fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7426 if (RetVT.SimpleTy != MVT::v8i16)
7427 return Register();
7428 if ((Subtarget->isNeonAvailable())) {
7429 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULLv8i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
7430 }
7431 return Register();
7432}
7433
7434Register fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7435 if (RetVT.SimpleTy != MVT::v16i8)
7436 return Register();
7437 if ((Subtarget->hasAES())) {
7438 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULLv1i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7439 }
7440 return Register();
7441}
7442
7443Register fastEmit_AArch64ISD_PMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7444 switch (VT.SimpleTy) {
7445 case MVT::v8i8: return fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
7446 case MVT::v1i64: return fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(RetVT, Op0, Op1);
7447 default: return Register();
7448 }
7449}
7450
7451// FastEmit functions for AArch64ISD::SMULL.
7452
7453Register fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7454 if (RetVT.SimpleTy != MVT::v8i16)
7455 return Register();
7456 if ((Subtarget->isNeonAvailable())) {
7457 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv8i8_v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7458 }
7459 return Register();
7460}
7461
7462Register fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7463 if (RetVT.SimpleTy != MVT::v4i32)
7464 return Register();
7465 if ((Subtarget->isNeonAvailable())) {
7466 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv4i16_v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7467 }
7468 return Register();
7469}
7470
7471Register fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7472 if (RetVT.SimpleTy != MVT::v2i64)
7473 return Register();
7474 if ((Subtarget->isNeonAvailable())) {
7475 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv2i32_v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7476 }
7477 return Register();
7478}
7479
7480Register fastEmit_AArch64ISD_SMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7481 switch (VT.SimpleTy) {
7482 case MVT::v8i8: return fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
7483 case MVT::v4i16: return fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(RetVT, Op0, Op1);
7484 case MVT::v2i32: return fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(RetVT, Op0, Op1);
7485 default: return Register();
7486 }
7487}
7488
7489// FastEmit functions for AArch64ISD::SQADD.
7490
7491Register fastEmit_AArch64ISD_SQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7492 if (RetVT.SimpleTy != MVT::f32)
7493 return Register();
7494 if ((Subtarget->isNeonAvailable())) {
7495 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7496 }
7497 return Register();
7498}
7499
7500Register fastEmit_AArch64ISD_SQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7501 if (RetVT.SimpleTy != MVT::f64)
7502 return Register();
7503 if ((Subtarget->isNeonAvailable())) {
7504 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7505 }
7506 return Register();
7507}
7508
7509Register fastEmit_AArch64ISD_SQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7510 switch (VT.SimpleTy) {
7511 case MVT::f32: return fastEmit_AArch64ISD_SQADD_MVT_f32_rr(RetVT, Op0, Op1);
7512 case MVT::f64: return fastEmit_AArch64ISD_SQADD_MVT_f64_rr(RetVT, Op0, Op1);
7513 default: return Register();
7514 }
7515}
7516
7517// FastEmit functions for AArch64ISD::SQDMULH.
7518
7519Register fastEmit_AArch64ISD_SQDMULH_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7520 if (RetVT.SimpleTy != MVT::f32)
7521 return Register();
7522 if ((Subtarget->isNeonAvailable())) {
7523 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7524 }
7525 return Register();
7526}
7527
7528Register fastEmit_AArch64ISD_SQDMULH_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7529 if (RetVT.SimpleTy != MVT::v4i16)
7530 return Register();
7531 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7532}
7533
7534Register fastEmit_AArch64ISD_SQDMULH_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7535 if (RetVT.SimpleTy != MVT::v8i16)
7536 return Register();
7537 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7538}
7539
7540Register fastEmit_AArch64ISD_SQDMULH_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7541 if (RetVT.SimpleTy != MVT::v2i32)
7542 return Register();
7543 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7544}
7545
7546Register fastEmit_AArch64ISD_SQDMULH_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7547 if (RetVT.SimpleTy != MVT::v4i32)
7548 return Register();
7549 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7550}
7551
7552Register fastEmit_AArch64ISD_SQDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7553 switch (VT.SimpleTy) {
7554 case MVT::f32: return fastEmit_AArch64ISD_SQDMULH_MVT_f32_rr(RetVT, Op0, Op1);
7555 case MVT::v4i16: return fastEmit_AArch64ISD_SQDMULH_MVT_v4i16_rr(RetVT, Op0, Op1);
7556 case MVT::v8i16: return fastEmit_AArch64ISD_SQDMULH_MVT_v8i16_rr(RetVT, Op0, Op1);
7557 case MVT::v2i32: return fastEmit_AArch64ISD_SQDMULH_MVT_v2i32_rr(RetVT, Op0, Op1);
7558 case MVT::v4i32: return fastEmit_AArch64ISD_SQDMULH_MVT_v4i32_rr(RetVT, Op0, Op1);
7559 default: return Register();
7560 }
7561}
7562
7563// FastEmit functions for AArch64ISD::SQDMULL.
7564
7565Register fastEmit_AArch64ISD_SQDMULL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7566 if (RetVT.SimpleTy != MVT::f64)
7567 return Register();
7568 if ((Subtarget->isNeonAvailable())) {
7569 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULLi32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7570 }
7571 return Register();
7572}
7573
7574Register fastEmit_AArch64ISD_SQDMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7575 switch (VT.SimpleTy) {
7576 case MVT::f32: return fastEmit_AArch64ISD_SQDMULL_MVT_f32_rr(RetVT, Op0, Op1);
7577 default: return Register();
7578 }
7579}
7580
7581// FastEmit functions for AArch64ISD::SQRDMULH.
7582
7583Register fastEmit_AArch64ISD_SQRDMULH_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7584 if (RetVT.SimpleTy != MVT::f32)
7585 return Register();
7586 if ((Subtarget->isNeonAvailable())) {
7587 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRDMULHv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7588 }
7589 return Register();
7590}
7591
7592Register fastEmit_AArch64ISD_SQRDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7593 switch (VT.SimpleTy) {
7594 case MVT::f32: return fastEmit_AArch64ISD_SQRDMULH_MVT_f32_rr(RetVT, Op0, Op1);
7595 default: return Register();
7596 }
7597}
7598
7599// FastEmit functions for AArch64ISD::SQRSHL.
7600
7601Register fastEmit_AArch64ISD_SQRSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7602 if (RetVT.SimpleTy != MVT::f32)
7603 return Register();
7604 if ((Subtarget->isNeonAvailable())) {
7605 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7606 }
7607 return Register();
7608}
7609
7610Register fastEmit_AArch64ISD_SQRSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7611 if (RetVT.SimpleTy != MVT::f64)
7612 return Register();
7613 if ((Subtarget->isNeonAvailable())) {
7614 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7615 }
7616 return Register();
7617}
7618
7619Register fastEmit_AArch64ISD_SQRSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7620 switch (VT.SimpleTy) {
7621 case MVT::f32: return fastEmit_AArch64ISD_SQRSHL_MVT_f32_rr(RetVT, Op0, Op1);
7622 case MVT::f64: return fastEmit_AArch64ISD_SQRSHL_MVT_f64_rr(RetVT, Op0, Op1);
7623 default: return Register();
7624 }
7625}
7626
7627// FastEmit functions for AArch64ISD::SQSHL.
7628
7629Register fastEmit_AArch64ISD_SQSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7630 if (RetVT.SimpleTy != MVT::f32)
7631 return Register();
7632 if ((Subtarget->isNeonAvailable())) {
7633 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7634 }
7635 return Register();
7636}
7637
7638Register fastEmit_AArch64ISD_SQSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7639 if (RetVT.SimpleTy != MVT::f64)
7640 return Register();
7641 if ((Subtarget->isNeonAvailable())) {
7642 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7643 }
7644 return Register();
7645}
7646
7647Register fastEmit_AArch64ISD_SQSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7648 switch (VT.SimpleTy) {
7649 case MVT::f32: return fastEmit_AArch64ISD_SQSHL_MVT_f32_rr(RetVT, Op0, Op1);
7650 case MVT::f64: return fastEmit_AArch64ISD_SQSHL_MVT_f64_rr(RetVT, Op0, Op1);
7651 default: return Register();
7652 }
7653}
7654
7655// FastEmit functions for AArch64ISD::SQSUB.
7656
7657Register fastEmit_AArch64ISD_SQSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7658 if (RetVT.SimpleTy != MVT::f32)
7659 return Register();
7660 if ((Subtarget->isNeonAvailable())) {
7661 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7662 }
7663 return Register();
7664}
7665
7666Register fastEmit_AArch64ISD_SQSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7667 if (RetVT.SimpleTy != MVT::f64)
7668 return Register();
7669 if ((Subtarget->isNeonAvailable())) {
7670 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7671 }
7672 return Register();
7673}
7674
7675Register fastEmit_AArch64ISD_SQSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7676 switch (VT.SimpleTy) {
7677 case MVT::f32: return fastEmit_AArch64ISD_SQSUB_MVT_f32_rr(RetVT, Op0, Op1);
7678 case MVT::f64: return fastEmit_AArch64ISD_SQSUB_MVT_f64_rr(RetVT, Op0, Op1);
7679 default: return Register();
7680 }
7681}
7682
7683// FastEmit functions for AArch64ISD::SUQADD.
7684
7685Register fastEmit_AArch64ISD_SUQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7686 if (RetVT.SimpleTy != MVT::f32)
7687 return Register();
7688 if ((Subtarget->isNeonAvailable())) {
7689 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7690 }
7691 return Register();
7692}
7693
7694Register fastEmit_AArch64ISD_SUQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7695 if (RetVT.SimpleTy != MVT::f64)
7696 return Register();
7697 if ((Subtarget->isNeonAvailable())) {
7698 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7699 }
7700 return Register();
7701}
7702
7703Register fastEmit_AArch64ISD_SUQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7704 switch (VT.SimpleTy) {
7705 case MVT::f32: return fastEmit_AArch64ISD_SUQADD_MVT_f32_rr(RetVT, Op0, Op1);
7706 case MVT::f64: return fastEmit_AArch64ISD_SUQADD_MVT_f64_rr(RetVT, Op0, Op1);
7707 default: return Register();
7708 }
7709}
7710
7711// FastEmit functions for AArch64ISD::TBL.
7712
7713Register fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7714 if (RetVT.SimpleTy != MVT::nxv16i8)
7715 return Register();
7716 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7717 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
7718 }
7719 return Register();
7720}
7721
7722Register fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7723 if (RetVT.SimpleTy != MVT::nxv8i16)
7724 return Register();
7725 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7726 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7727 }
7728 return Register();
7729}
7730
7731Register fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7732 if (RetVT.SimpleTy != MVT::nxv4i32)
7733 return Register();
7734 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7735 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7736 }
7737 return Register();
7738}
7739
7740Register fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7741 if (RetVT.SimpleTy != MVT::nxv2i64)
7742 return Register();
7743 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7744 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7745 }
7746 return Register();
7747}
7748
7749Register fastEmit_AArch64ISD_TBL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7750 switch (VT.SimpleTy) {
7751 case MVT::nxv16i8: return fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(RetVT, Op0, Op1);
7752 case MVT::nxv8i16: return fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(RetVT, Op0, Op1);
7753 case MVT::nxv4i32: return fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(RetVT, Op0, Op1);
7754 case MVT::nxv2i64: return fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(RetVT, Op0, Op1);
7755 default: return Register();
7756 }
7757}
7758
7759// FastEmit functions for AArch64ISD::TRN1.
7760
7761Register fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7762 if (RetVT.SimpleTy != MVT::v8i8)
7763 return Register();
7764 if ((Subtarget->isNeonAvailable())) {
7765 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
7766 }
7767 return Register();
7768}
7769
7770Register fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7771 if (RetVT.SimpleTy != MVT::v16i8)
7772 return Register();
7773 if ((Subtarget->isNeonAvailable())) {
7774 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
7775 }
7776 return Register();
7777}
7778
7779Register fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7780 if (RetVT.SimpleTy != MVT::v4i16)
7781 return Register();
7782 if ((Subtarget->isNeonAvailable())) {
7783 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7784 }
7785 return Register();
7786}
7787
7788Register fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7789 if (RetVT.SimpleTy != MVT::v8i16)
7790 return Register();
7791 if ((Subtarget->isNeonAvailable())) {
7792 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7793 }
7794 return Register();
7795}
7796
7797Register fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7798 if (RetVT.SimpleTy != MVT::v2i32)
7799 return Register();
7800 if ((Subtarget->isNeonAvailable())) {
7801 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7802 }
7803 return Register();
7804}
7805
7806Register fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7807 if (RetVT.SimpleTy != MVT::v4i32)
7808 return Register();
7809 if ((Subtarget->isNeonAvailable())) {
7810 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7811 }
7812 return Register();
7813}
7814
7815Register fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7816 if (RetVT.SimpleTy != MVT::v2i64)
7817 return Register();
7818 if ((Subtarget->isNeonAvailable())) {
7819 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7820 }
7821 return Register();
7822}
7823
7824Register fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7825 if (RetVT.SimpleTy != MVT::v4f16)
7826 return Register();
7827 if ((Subtarget->isNeonAvailable())) {
7828 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7829 }
7830 return Register();
7831}
7832
7833Register fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7834 if (RetVT.SimpleTy != MVT::v8f16)
7835 return Register();
7836 if ((Subtarget->isNeonAvailable())) {
7837 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7838 }
7839 return Register();
7840}
7841
7842Register fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
7843 if (RetVT.SimpleTy != MVT::v4bf16)
7844 return Register();
7845 if ((Subtarget->isNeonAvailable())) {
7846 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7847 }
7848 return Register();
7849}
7850
7851Register fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
7852 if (RetVT.SimpleTy != MVT::v8bf16)
7853 return Register();
7854 if ((Subtarget->isNeonAvailable())) {
7855 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7856 }
7857 return Register();
7858}
7859
7860Register fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7861 if (RetVT.SimpleTy != MVT::v2f32)
7862 return Register();
7863 if ((Subtarget->isNeonAvailable())) {
7864 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7865 }
7866 return Register();
7867}
7868
7869Register fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7870 if (RetVT.SimpleTy != MVT::v4f32)
7871 return Register();
7872 if ((Subtarget->isNeonAvailable())) {
7873 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7874 }
7875 return Register();
7876}
7877
7878Register fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7879 if (RetVT.SimpleTy != MVT::v2f64)
7880 return Register();
7881 if ((Subtarget->isNeonAvailable())) {
7882 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7883 }
7884 return Register();
7885}
7886
7887Register fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
7888 if (RetVT.SimpleTy != MVT::nxv2i1)
7889 return Register();
7890 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7891 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
7892 }
7893 return Register();
7894}
7895
7896Register fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
7897 if (RetVT.SimpleTy != MVT::nxv4i1)
7898 return Register();
7899 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7900 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
7901 }
7902 return Register();
7903}
7904
7905Register fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
7906 if (RetVT.SimpleTy != MVT::nxv8i1)
7907 return Register();
7908 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7909 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
7910 }
7911 return Register();
7912}
7913
7914Register fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7915 if (RetVT.SimpleTy != MVT::nxv16i1)
7916 return Register();
7917 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7918 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
7919 }
7920 return Register();
7921}
7922
7923Register fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7924 if (RetVT.SimpleTy != MVT::nxv16i8)
7925 return Register();
7926 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7927 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
7928 }
7929 return Register();
7930}
7931
7932Register fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7933 if (RetVT.SimpleTy != MVT::nxv8i16)
7934 return Register();
7935 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7936 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7937 }
7938 return Register();
7939}
7940
7941Register fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7942 if (RetVT.SimpleTy != MVT::nxv4i32)
7943 return Register();
7944 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7945 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7946 }
7947 return Register();
7948}
7949
7950Register fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7951 if (RetVT.SimpleTy != MVT::nxv2i64)
7952 return Register();
7953 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7954 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7955 }
7956 return Register();
7957}
7958
7959Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
7960 if (RetVT.SimpleTy != MVT::nxv2f16)
7961 return Register();
7962 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7963 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7964 }
7965 return Register();
7966}
7967
7968Register fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7969 if (RetVT.SimpleTy != MVT::nxv4f16)
7970 return Register();
7971 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7972 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7973 }
7974 return Register();
7975}
7976
7977Register fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7978 if (RetVT.SimpleTy != MVT::nxv8f16)
7979 return Register();
7980 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7981 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7982 }
7983 return Register();
7984}
7985
7986Register fastEmit_AArch64ISD_TRN1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
7987 if (RetVT.SimpleTy != MVT::nxv2bf16)
7988 return Register();
7989 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7990 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7991 }
7992 return Register();
7993}
7994
7995Register fastEmit_AArch64ISD_TRN1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
7996 if (RetVT.SimpleTy != MVT::nxv4bf16)
7997 return Register();
7998 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7999 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8000 }
8001 return Register();
8002}
8003
8004Register fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8005 if (RetVT.SimpleTy != MVT::nxv8bf16)
8006 return Register();
8007 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8008 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8009 }
8010 return Register();
8011}
8012
8013Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8014 if (RetVT.SimpleTy != MVT::nxv2f32)
8015 return Register();
8016 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8017 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8018 }
8019 return Register();
8020}
8021
8022Register fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8023 if (RetVT.SimpleTy != MVT::nxv4f32)
8024 return Register();
8025 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8026 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8027 }
8028 return Register();
8029}
8030
8031Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8032 if (RetVT.SimpleTy != MVT::nxv2f64)
8033 return Register();
8034 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8035 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8036 }
8037 return Register();
8038}
8039
8040Register fastEmit_AArch64ISD_TRN1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8041 switch (VT.SimpleTy) {
8042 case MVT::v8i8: return fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(RetVT, Op0, Op1);
8043 case MVT::v16i8: return fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(RetVT, Op0, Op1);
8044 case MVT::v4i16: return fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(RetVT, Op0, Op1);
8045 case MVT::v8i16: return fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(RetVT, Op0, Op1);
8046 case MVT::v2i32: return fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(RetVT, Op0, Op1);
8047 case MVT::v4i32: return fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(RetVT, Op0, Op1);
8048 case MVT::v2i64: return fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(RetVT, Op0, Op1);
8049 case MVT::v4f16: return fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(RetVT, Op0, Op1);
8050 case MVT::v8f16: return fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(RetVT, Op0, Op1);
8051 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(RetVT, Op0, Op1);
8052 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(RetVT, Op0, Op1);
8053 case MVT::v2f32: return fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(RetVT, Op0, Op1);
8054 case MVT::v4f32: return fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(RetVT, Op0, Op1);
8055 case MVT::v2f64: return fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(RetVT, Op0, Op1);
8056 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8057 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8058 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8059 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8060 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8061 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8062 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8063 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8064 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8065 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8066 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8067 case MVT::nxv2bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8068 case MVT::nxv4bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8069 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8070 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8071 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8072 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8073 default: return Register();
8074 }
8075}
8076
8077// FastEmit functions for AArch64ISD::TRN2.
8078
8079Register fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8080 if (RetVT.SimpleTy != MVT::v8i8)
8081 return Register();
8082 if ((Subtarget->isNeonAvailable())) {
8083 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8084 }
8085 return Register();
8086}
8087
8088Register fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8089 if (RetVT.SimpleTy != MVT::v16i8)
8090 return Register();
8091 if ((Subtarget->isNeonAvailable())) {
8092 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8093 }
8094 return Register();
8095}
8096
8097Register fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8098 if (RetVT.SimpleTy != MVT::v4i16)
8099 return Register();
8100 if ((Subtarget->isNeonAvailable())) {
8101 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8102 }
8103 return Register();
8104}
8105
8106Register fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8107 if (RetVT.SimpleTy != MVT::v8i16)
8108 return Register();
8109 if ((Subtarget->isNeonAvailable())) {
8110 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8111 }
8112 return Register();
8113}
8114
8115Register fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8116 if (RetVT.SimpleTy != MVT::v2i32)
8117 return Register();
8118 if ((Subtarget->isNeonAvailable())) {
8119 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8120 }
8121 return Register();
8122}
8123
8124Register fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8125 if (RetVT.SimpleTy != MVT::v4i32)
8126 return Register();
8127 if ((Subtarget->isNeonAvailable())) {
8128 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8129 }
8130 return Register();
8131}
8132
8133Register fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8134 if (RetVT.SimpleTy != MVT::v2i64)
8135 return Register();
8136 if ((Subtarget->isNeonAvailable())) {
8137 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8138 }
8139 return Register();
8140}
8141
8142Register fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8143 if (RetVT.SimpleTy != MVT::v4f16)
8144 return Register();
8145 if ((Subtarget->isNeonAvailable())) {
8146 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8147 }
8148 return Register();
8149}
8150
8151Register fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8152 if (RetVT.SimpleTy != MVT::v8f16)
8153 return Register();
8154 if ((Subtarget->isNeonAvailable())) {
8155 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8156 }
8157 return Register();
8158}
8159
8160Register fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8161 if (RetVT.SimpleTy != MVT::v4bf16)
8162 return Register();
8163 if ((Subtarget->isNeonAvailable())) {
8164 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8165 }
8166 return Register();
8167}
8168
8169Register fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8170 if (RetVT.SimpleTy != MVT::v8bf16)
8171 return Register();
8172 if ((Subtarget->isNeonAvailable())) {
8173 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8174 }
8175 return Register();
8176}
8177
8178Register fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8179 if (RetVT.SimpleTy != MVT::v2f32)
8180 return Register();
8181 if ((Subtarget->isNeonAvailable())) {
8182 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8183 }
8184 return Register();
8185}
8186
8187Register fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8188 if (RetVT.SimpleTy != MVT::v4f32)
8189 return Register();
8190 if ((Subtarget->isNeonAvailable())) {
8191 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8192 }
8193 return Register();
8194}
8195
8196Register fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8197 if (RetVT.SimpleTy != MVT::v2f64)
8198 return Register();
8199 if ((Subtarget->isNeonAvailable())) {
8200 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8201 }
8202 return Register();
8203}
8204
8205Register fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8206 if (RetVT.SimpleTy != MVT::nxv2i1)
8207 return Register();
8208 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8209 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8210 }
8211 return Register();
8212}
8213
8214Register fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8215 if (RetVT.SimpleTy != MVT::nxv4i1)
8216 return Register();
8217 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8218 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8219 }
8220 return Register();
8221}
8222
8223Register fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8224 if (RetVT.SimpleTy != MVT::nxv8i1)
8225 return Register();
8226 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8227 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8228 }
8229 return Register();
8230}
8231
8232Register fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8233 if (RetVT.SimpleTy != MVT::nxv16i1)
8234 return Register();
8235 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8236 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8237 }
8238 return Register();
8239}
8240
8241Register fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8242 if (RetVT.SimpleTy != MVT::nxv16i8)
8243 return Register();
8244 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8245 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8246 }
8247 return Register();
8248}
8249
8250Register fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8251 if (RetVT.SimpleTy != MVT::nxv8i16)
8252 return Register();
8253 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8254 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8255 }
8256 return Register();
8257}
8258
8259Register fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8260 if (RetVT.SimpleTy != MVT::nxv4i32)
8261 return Register();
8262 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8263 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8264 }
8265 return Register();
8266}
8267
8268Register fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8269 if (RetVT.SimpleTy != MVT::nxv2i64)
8270 return Register();
8271 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8272 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8273 }
8274 return Register();
8275}
8276
8277Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8278 if (RetVT.SimpleTy != MVT::nxv2f16)
8279 return Register();
8280 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8281 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8282 }
8283 return Register();
8284}
8285
8286Register fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8287 if (RetVT.SimpleTy != MVT::nxv4f16)
8288 return Register();
8289 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8290 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8291 }
8292 return Register();
8293}
8294
8295Register fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8296 if (RetVT.SimpleTy != MVT::nxv8f16)
8297 return Register();
8298 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8299 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8300 }
8301 return Register();
8302}
8303
8304Register fastEmit_AArch64ISD_TRN2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8305 if (RetVT.SimpleTy != MVT::nxv2bf16)
8306 return Register();
8307 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8308 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8309 }
8310 return Register();
8311}
8312
8313Register fastEmit_AArch64ISD_TRN2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8314 if (RetVT.SimpleTy != MVT::nxv4bf16)
8315 return Register();
8316 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8317 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8318 }
8319 return Register();
8320}
8321
8322Register fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8323 if (RetVT.SimpleTy != MVT::nxv8bf16)
8324 return Register();
8325 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8326 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8327 }
8328 return Register();
8329}
8330
8331Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8332 if (RetVT.SimpleTy != MVT::nxv2f32)
8333 return Register();
8334 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8335 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8336 }
8337 return Register();
8338}
8339
8340Register fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8341 if (RetVT.SimpleTy != MVT::nxv4f32)
8342 return Register();
8343 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8344 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8345 }
8346 return Register();
8347}
8348
8349Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8350 if (RetVT.SimpleTy != MVT::nxv2f64)
8351 return Register();
8352 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8353 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8354 }
8355 return Register();
8356}
8357
8358Register fastEmit_AArch64ISD_TRN2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8359 switch (VT.SimpleTy) {
8360 case MVT::v8i8: return fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(RetVT, Op0, Op1);
8361 case MVT::v16i8: return fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(RetVT, Op0, Op1);
8362 case MVT::v4i16: return fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(RetVT, Op0, Op1);
8363 case MVT::v8i16: return fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(RetVT, Op0, Op1);
8364 case MVT::v2i32: return fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(RetVT, Op0, Op1);
8365 case MVT::v4i32: return fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(RetVT, Op0, Op1);
8366 case MVT::v2i64: return fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(RetVT, Op0, Op1);
8367 case MVT::v4f16: return fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(RetVT, Op0, Op1);
8368 case MVT::v8f16: return fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(RetVT, Op0, Op1);
8369 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(RetVT, Op0, Op1);
8370 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(RetVT, Op0, Op1);
8371 case MVT::v2f32: return fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(RetVT, Op0, Op1);
8372 case MVT::v4f32: return fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(RetVT, Op0, Op1);
8373 case MVT::v2f64: return fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(RetVT, Op0, Op1);
8374 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8375 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8376 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8377 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8378 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8379 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8380 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8381 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8382 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8383 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8384 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8385 case MVT::nxv2bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8386 case MVT::nxv4bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8387 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8388 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8389 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8390 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8391 default: return Register();
8392 }
8393}
8394
8395// FastEmit functions for AArch64ISD::UMULL.
8396
8397Register fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8398 if (RetVT.SimpleTy != MVT::v8i16)
8399 return Register();
8400 if ((Subtarget->isNeonAvailable())) {
8401 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv8i8_v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8402 }
8403 return Register();
8404}
8405
8406Register fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8407 if (RetVT.SimpleTy != MVT::v4i32)
8408 return Register();
8409 if ((Subtarget->isNeonAvailable())) {
8410 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv4i16_v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8411 }
8412 return Register();
8413}
8414
8415Register fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8416 if (RetVT.SimpleTy != MVT::v2i64)
8417 return Register();
8418 if ((Subtarget->isNeonAvailable())) {
8419 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv2i32_v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8420 }
8421 return Register();
8422}
8423
8424Register fastEmit_AArch64ISD_UMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8425 switch (VT.SimpleTy) {
8426 case MVT::v8i8: return fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
8427 case MVT::v4i16: return fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(RetVT, Op0, Op1);
8428 case MVT::v2i32: return fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(RetVT, Op0, Op1);
8429 default: return Register();
8430 }
8431}
8432
8433// FastEmit functions for AArch64ISD::UQADD.
8434
8435Register fastEmit_AArch64ISD_UQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8436 if (RetVT.SimpleTy != MVT::f32)
8437 return Register();
8438 if ((Subtarget->isNeonAvailable())) {
8439 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8440 }
8441 return Register();
8442}
8443
8444Register fastEmit_AArch64ISD_UQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8445 if (RetVT.SimpleTy != MVT::f64)
8446 return Register();
8447 if ((Subtarget->isNeonAvailable())) {
8448 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8449 }
8450 return Register();
8451}
8452
8453Register fastEmit_AArch64ISD_UQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8454 switch (VT.SimpleTy) {
8455 case MVT::f32: return fastEmit_AArch64ISD_UQADD_MVT_f32_rr(RetVT, Op0, Op1);
8456 case MVT::f64: return fastEmit_AArch64ISD_UQADD_MVT_f64_rr(RetVT, Op0, Op1);
8457 default: return Register();
8458 }
8459}
8460
8461// FastEmit functions for AArch64ISD::UQRSHL.
8462
8463Register fastEmit_AArch64ISD_UQRSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8464 if (RetVT.SimpleTy != MVT::f32)
8465 return Register();
8466 if ((Subtarget->isNeonAvailable())) {
8467 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQRSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8468 }
8469 return Register();
8470}
8471
8472Register fastEmit_AArch64ISD_UQRSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8473 if (RetVT.SimpleTy != MVT::f64)
8474 return Register();
8475 if ((Subtarget->isNeonAvailable())) {
8476 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQRSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8477 }
8478 return Register();
8479}
8480
8481Register fastEmit_AArch64ISD_UQRSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8482 switch (VT.SimpleTy) {
8483 case MVT::f32: return fastEmit_AArch64ISD_UQRSHL_MVT_f32_rr(RetVT, Op0, Op1);
8484 case MVT::f64: return fastEmit_AArch64ISD_UQRSHL_MVT_f64_rr(RetVT, Op0, Op1);
8485 default: return Register();
8486 }
8487}
8488
8489// FastEmit functions for AArch64ISD::UQSHL.
8490
8491Register fastEmit_AArch64ISD_UQSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8492 if (RetVT.SimpleTy != MVT::f32)
8493 return Register();
8494 if ((Subtarget->isNeonAvailable())) {
8495 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8496 }
8497 return Register();
8498}
8499
8500Register fastEmit_AArch64ISD_UQSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8501 if (RetVT.SimpleTy != MVT::f64)
8502 return Register();
8503 if ((Subtarget->isNeonAvailable())) {
8504 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8505 }
8506 return Register();
8507}
8508
8509Register fastEmit_AArch64ISD_UQSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8510 switch (VT.SimpleTy) {
8511 case MVT::f32: return fastEmit_AArch64ISD_UQSHL_MVT_f32_rr(RetVT, Op0, Op1);
8512 case MVT::f64: return fastEmit_AArch64ISD_UQSHL_MVT_f64_rr(RetVT, Op0, Op1);
8513 default: return Register();
8514 }
8515}
8516
8517// FastEmit functions for AArch64ISD::UQSUB.
8518
8519Register fastEmit_AArch64ISD_UQSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8520 if (RetVT.SimpleTy != MVT::f32)
8521 return Register();
8522 if ((Subtarget->isNeonAvailable())) {
8523 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8524 }
8525 return Register();
8526}
8527
8528Register fastEmit_AArch64ISD_UQSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8529 if (RetVT.SimpleTy != MVT::f64)
8530 return Register();
8531 if ((Subtarget->isNeonAvailable())) {
8532 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8533 }
8534 return Register();
8535}
8536
8537Register fastEmit_AArch64ISD_UQSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8538 switch (VT.SimpleTy) {
8539 case MVT::f32: return fastEmit_AArch64ISD_UQSUB_MVT_f32_rr(RetVT, Op0, Op1);
8540 case MVT::f64: return fastEmit_AArch64ISD_UQSUB_MVT_f64_rr(RetVT, Op0, Op1);
8541 default: return Register();
8542 }
8543}
8544
8545// FastEmit functions for AArch64ISD::USQADD.
8546
8547Register fastEmit_AArch64ISD_USQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8548 if (RetVT.SimpleTy != MVT::f32)
8549 return Register();
8550 if ((Subtarget->isNeonAvailable())) {
8551 return fastEmitInst_rr(MachineInstOpcode: AArch64::USQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8552 }
8553 return Register();
8554}
8555
8556Register fastEmit_AArch64ISD_USQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8557 if (RetVT.SimpleTy != MVT::f64)
8558 return Register();
8559 if ((Subtarget->isNeonAvailable())) {
8560 return fastEmitInst_rr(MachineInstOpcode: AArch64::USQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8561 }
8562 return Register();
8563}
8564
8565Register fastEmit_AArch64ISD_USQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8566 switch (VT.SimpleTy) {
8567 case MVT::f32: return fastEmit_AArch64ISD_USQADD_MVT_f32_rr(RetVT, Op0, Op1);
8568 case MVT::f64: return fastEmit_AArch64ISD_USQADD_MVT_f64_rr(RetVT, Op0, Op1);
8569 default: return Register();
8570 }
8571}
8572
8573// FastEmit functions for AArch64ISD::UZP1.
8574
8575Register fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8576 if (RetVT.SimpleTy != MVT::v8i8)
8577 return Register();
8578 if ((Subtarget->isNeonAvailable())) {
8579 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8580 }
8581 return Register();
8582}
8583
8584Register fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8585 if (RetVT.SimpleTy != MVT::v16i8)
8586 return Register();
8587 if ((Subtarget->isNeonAvailable())) {
8588 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8589 }
8590 return Register();
8591}
8592
8593Register fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8594 if (RetVT.SimpleTy != MVT::v4i16)
8595 return Register();
8596 if ((Subtarget->isNeonAvailable())) {
8597 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8598 }
8599 return Register();
8600}
8601
8602Register fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8603 if (RetVT.SimpleTy != MVT::v8i16)
8604 return Register();
8605 if ((Subtarget->isNeonAvailable())) {
8606 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8607 }
8608 return Register();
8609}
8610
8611Register fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8612 if (RetVT.SimpleTy != MVT::v2i32)
8613 return Register();
8614 if ((Subtarget->isNeonAvailable())) {
8615 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8616 }
8617 return Register();
8618}
8619
8620Register fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8621 if (RetVT.SimpleTy != MVT::v4i32)
8622 return Register();
8623 if ((Subtarget->isNeonAvailable())) {
8624 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8625 }
8626 return Register();
8627}
8628
8629Register fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8630 if (RetVT.SimpleTy != MVT::v2i64)
8631 return Register();
8632 if ((Subtarget->isNeonAvailable())) {
8633 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8634 }
8635 return Register();
8636}
8637
8638Register fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8639 if (RetVT.SimpleTy != MVT::v4f16)
8640 return Register();
8641 if ((Subtarget->isNeonAvailable())) {
8642 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8643 }
8644 return Register();
8645}
8646
8647Register fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8648 if (RetVT.SimpleTy != MVT::v8f16)
8649 return Register();
8650 if ((Subtarget->isNeonAvailable())) {
8651 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8652 }
8653 return Register();
8654}
8655
8656Register fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8657 if (RetVT.SimpleTy != MVT::v4bf16)
8658 return Register();
8659 if ((Subtarget->isNeonAvailable())) {
8660 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8661 }
8662 return Register();
8663}
8664
8665Register fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8666 if (RetVT.SimpleTy != MVT::v8bf16)
8667 return Register();
8668 if ((Subtarget->isNeonAvailable())) {
8669 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8670 }
8671 return Register();
8672}
8673
8674Register fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8675 if (RetVT.SimpleTy != MVT::v2f32)
8676 return Register();
8677 if ((Subtarget->isNeonAvailable())) {
8678 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8679 }
8680 return Register();
8681}
8682
8683Register fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8684 if (RetVT.SimpleTy != MVT::v4f32)
8685 return Register();
8686 if ((Subtarget->isNeonAvailable())) {
8687 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8688 }
8689 return Register();
8690}
8691
8692Register fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8693 if (RetVT.SimpleTy != MVT::v2f64)
8694 return Register();
8695 if ((Subtarget->isNeonAvailable())) {
8696 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8697 }
8698 return Register();
8699}
8700
8701Register fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8702 if (RetVT.SimpleTy != MVT::nxv2i1)
8703 return Register();
8704 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8705 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8706 }
8707 return Register();
8708}
8709
8710Register fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8711 if (RetVT.SimpleTy != MVT::nxv4i1)
8712 return Register();
8713 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8714 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8715 }
8716 return Register();
8717}
8718
8719Register fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8720 if (RetVT.SimpleTy != MVT::nxv8i1)
8721 return Register();
8722 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8723 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8724 }
8725 return Register();
8726}
8727
8728Register fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8729 if (RetVT.SimpleTy != MVT::nxv16i1)
8730 return Register();
8731 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8732 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8733 }
8734 return Register();
8735}
8736
8737Register fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8738 if (RetVT.SimpleTy != MVT::nxv16i8)
8739 return Register();
8740 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8741 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8742 }
8743 return Register();
8744}
8745
8746Register fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8747 if (RetVT.SimpleTy != MVT::nxv8i16)
8748 return Register();
8749 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8750 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8751 }
8752 return Register();
8753}
8754
8755Register fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8756 if (RetVT.SimpleTy != MVT::nxv4i32)
8757 return Register();
8758 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8759 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8760 }
8761 return Register();
8762}
8763
8764Register fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8765 if (RetVT.SimpleTy != MVT::nxv2i64)
8766 return Register();
8767 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8768 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8769 }
8770 return Register();
8771}
8772
8773Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8774 if (RetVT.SimpleTy != MVT::nxv2f16)
8775 return Register();
8776 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8777 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8778 }
8779 return Register();
8780}
8781
8782Register fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8783 if (RetVT.SimpleTy != MVT::nxv4f16)
8784 return Register();
8785 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8786 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8787 }
8788 return Register();
8789}
8790
8791Register fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8792 if (RetVT.SimpleTy != MVT::nxv8f16)
8793 return Register();
8794 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8795 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8796 }
8797 return Register();
8798}
8799
8800Register fastEmit_AArch64ISD_UZP1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8801 if (RetVT.SimpleTy != MVT::nxv2bf16)
8802 return Register();
8803 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8804 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8805 }
8806 return Register();
8807}
8808
8809Register fastEmit_AArch64ISD_UZP1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8810 if (RetVT.SimpleTy != MVT::nxv4bf16)
8811 return Register();
8812 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8813 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8814 }
8815 return Register();
8816}
8817
8818Register fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8819 if (RetVT.SimpleTy != MVT::nxv8bf16)
8820 return Register();
8821 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8822 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8823 }
8824 return Register();
8825}
8826
8827Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8828 if (RetVT.SimpleTy != MVT::nxv2f32)
8829 return Register();
8830 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8831 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8832 }
8833 return Register();
8834}
8835
8836Register fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8837 if (RetVT.SimpleTy != MVT::nxv4f32)
8838 return Register();
8839 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8840 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8841 }
8842 return Register();
8843}
8844
8845Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8846 if (RetVT.SimpleTy != MVT::nxv2f64)
8847 return Register();
8848 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8849 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8850 }
8851 return Register();
8852}
8853
8854Register fastEmit_AArch64ISD_UZP1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8855 switch (VT.SimpleTy) {
8856 case MVT::v8i8: return fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(RetVT, Op0, Op1);
8857 case MVT::v16i8: return fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(RetVT, Op0, Op1);
8858 case MVT::v4i16: return fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(RetVT, Op0, Op1);
8859 case MVT::v8i16: return fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(RetVT, Op0, Op1);
8860 case MVT::v2i32: return fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(RetVT, Op0, Op1);
8861 case MVT::v4i32: return fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(RetVT, Op0, Op1);
8862 case MVT::v2i64: return fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(RetVT, Op0, Op1);
8863 case MVT::v4f16: return fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(RetVT, Op0, Op1);
8864 case MVT::v8f16: return fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(RetVT, Op0, Op1);
8865 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(RetVT, Op0, Op1);
8866 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(RetVT, Op0, Op1);
8867 case MVT::v2f32: return fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(RetVT, Op0, Op1);
8868 case MVT::v4f32: return fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(RetVT, Op0, Op1);
8869 case MVT::v2f64: return fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(RetVT, Op0, Op1);
8870 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8871 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8872 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8873 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8874 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8875 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8876 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8877 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8878 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8879 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8880 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8881 case MVT::nxv2bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8882 case MVT::nxv4bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8883 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8884 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8885 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8886 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8887 default: return Register();
8888 }
8889}
8890
8891// FastEmit functions for AArch64ISD::UZP2.
8892
8893Register fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8894 if (RetVT.SimpleTy != MVT::v8i8)
8895 return Register();
8896 if ((Subtarget->isNeonAvailable())) {
8897 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8898 }
8899 return Register();
8900}
8901
8902Register fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8903 if (RetVT.SimpleTy != MVT::v16i8)
8904 return Register();
8905 if ((Subtarget->isNeonAvailable())) {
8906 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8907 }
8908 return Register();
8909}
8910
8911Register fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8912 if (RetVT.SimpleTy != MVT::v4i16)
8913 return Register();
8914 if ((Subtarget->isNeonAvailable())) {
8915 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8916 }
8917 return Register();
8918}
8919
8920Register fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8921 if (RetVT.SimpleTy != MVT::v8i16)
8922 return Register();
8923 if ((Subtarget->isNeonAvailable())) {
8924 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8925 }
8926 return Register();
8927}
8928
8929Register fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8930 if (RetVT.SimpleTy != MVT::v2i32)
8931 return Register();
8932 if ((Subtarget->isNeonAvailable())) {
8933 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8934 }
8935 return Register();
8936}
8937
8938Register fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8939 if (RetVT.SimpleTy != MVT::v4i32)
8940 return Register();
8941 if ((Subtarget->isNeonAvailable())) {
8942 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8943 }
8944 return Register();
8945}
8946
8947Register fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8948 if (RetVT.SimpleTy != MVT::v2i64)
8949 return Register();
8950 if ((Subtarget->isNeonAvailable())) {
8951 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8952 }
8953 return Register();
8954}
8955
8956Register fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8957 if (RetVT.SimpleTy != MVT::v4f16)
8958 return Register();
8959 if ((Subtarget->isNeonAvailable())) {
8960 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8961 }
8962 return Register();
8963}
8964
8965Register fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8966 if (RetVT.SimpleTy != MVT::v8f16)
8967 return Register();
8968 if ((Subtarget->isNeonAvailable())) {
8969 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8970 }
8971 return Register();
8972}
8973
8974Register fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8975 if (RetVT.SimpleTy != MVT::v4bf16)
8976 return Register();
8977 if ((Subtarget->isNeonAvailable())) {
8978 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8979 }
8980 return Register();
8981}
8982
8983Register fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8984 if (RetVT.SimpleTy != MVT::v8bf16)
8985 return Register();
8986 if ((Subtarget->isNeonAvailable())) {
8987 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8988 }
8989 return Register();
8990}
8991
8992Register fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8993 if (RetVT.SimpleTy != MVT::v2f32)
8994 return Register();
8995 if ((Subtarget->isNeonAvailable())) {
8996 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8997 }
8998 return Register();
8999}
9000
9001Register fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9002 if (RetVT.SimpleTy != MVT::v4f32)
9003 return Register();
9004 if ((Subtarget->isNeonAvailable())) {
9005 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9006 }
9007 return Register();
9008}
9009
9010Register fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9011 if (RetVT.SimpleTy != MVT::v2f64)
9012 return Register();
9013 if ((Subtarget->isNeonAvailable())) {
9014 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9015 }
9016 return Register();
9017}
9018
9019Register fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9020 if (RetVT.SimpleTy != MVT::nxv2i1)
9021 return Register();
9022 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9023 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9024 }
9025 return Register();
9026}
9027
9028Register fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9029 if (RetVT.SimpleTy != MVT::nxv4i1)
9030 return Register();
9031 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9032 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9033 }
9034 return Register();
9035}
9036
9037Register fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9038 if (RetVT.SimpleTy != MVT::nxv8i1)
9039 return Register();
9040 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9041 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9042 }
9043 return Register();
9044}
9045
9046Register fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9047 if (RetVT.SimpleTy != MVT::nxv16i1)
9048 return Register();
9049 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9050 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9051 }
9052 return Register();
9053}
9054
9055Register fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9056 if (RetVT.SimpleTy != MVT::nxv16i8)
9057 return Register();
9058 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9059 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9060 }
9061 return Register();
9062}
9063
9064Register fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9065 if (RetVT.SimpleTy != MVT::nxv8i16)
9066 return Register();
9067 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9068 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9069 }
9070 return Register();
9071}
9072
9073Register fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9074 if (RetVT.SimpleTy != MVT::nxv4i32)
9075 return Register();
9076 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9077 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9078 }
9079 return Register();
9080}
9081
9082Register fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9083 if (RetVT.SimpleTy != MVT::nxv2i64)
9084 return Register();
9085 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9086 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9087 }
9088 return Register();
9089}
9090
9091Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9092 if (RetVT.SimpleTy != MVT::nxv2f16)
9093 return Register();
9094 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9095 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9096 }
9097 return Register();
9098}
9099
9100Register fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9101 if (RetVT.SimpleTy != MVT::nxv4f16)
9102 return Register();
9103 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9104 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9105 }
9106 return Register();
9107}
9108
9109Register fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9110 if (RetVT.SimpleTy != MVT::nxv8f16)
9111 return Register();
9112 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9113 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9114 }
9115 return Register();
9116}
9117
9118Register fastEmit_AArch64ISD_UZP2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9119 if (RetVT.SimpleTy != MVT::nxv2bf16)
9120 return Register();
9121 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9122 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9123 }
9124 return Register();
9125}
9126
9127Register fastEmit_AArch64ISD_UZP2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9128 if (RetVT.SimpleTy != MVT::nxv4bf16)
9129 return Register();
9130 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9131 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9132 }
9133 return Register();
9134}
9135
9136Register fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9137 if (RetVT.SimpleTy != MVT::nxv8bf16)
9138 return Register();
9139 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9140 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9141 }
9142 return Register();
9143}
9144
9145Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9146 if (RetVT.SimpleTy != MVT::nxv2f32)
9147 return Register();
9148 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9149 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9150 }
9151 return Register();
9152}
9153
9154Register fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9155 if (RetVT.SimpleTy != MVT::nxv4f32)
9156 return Register();
9157 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9158 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9159 }
9160 return Register();
9161}
9162
9163Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9164 if (RetVT.SimpleTy != MVT::nxv2f64)
9165 return Register();
9166 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9167 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9168 }
9169 return Register();
9170}
9171
9172Register fastEmit_AArch64ISD_UZP2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9173 switch (VT.SimpleTy) {
9174 case MVT::v8i8: return fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(RetVT, Op0, Op1);
9175 case MVT::v16i8: return fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(RetVT, Op0, Op1);
9176 case MVT::v4i16: return fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(RetVT, Op0, Op1);
9177 case MVT::v8i16: return fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(RetVT, Op0, Op1);
9178 case MVT::v2i32: return fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(RetVT, Op0, Op1);
9179 case MVT::v4i32: return fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(RetVT, Op0, Op1);
9180 case MVT::v2i64: return fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(RetVT, Op0, Op1);
9181 case MVT::v4f16: return fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(RetVT, Op0, Op1);
9182 case MVT::v8f16: return fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(RetVT, Op0, Op1);
9183 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(RetVT, Op0, Op1);
9184 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(RetVT, Op0, Op1);
9185 case MVT::v2f32: return fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(RetVT, Op0, Op1);
9186 case MVT::v4f32: return fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(RetVT, Op0, Op1);
9187 case MVT::v2f64: return fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(RetVT, Op0, Op1);
9188 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9189 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9190 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9191 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9192 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9193 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9194 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9195 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9196 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9197 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9198 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9199 case MVT::nxv2bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9200 case MVT::nxv4bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9201 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9202 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9203 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9204 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9205 default: return Register();
9206 }
9207}
9208
9209// FastEmit functions for AArch64ISD::ZIP1.
9210
9211Register fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9212 if (RetVT.SimpleTy != MVT::v8i8)
9213 return Register();
9214 if ((Subtarget->isNeonAvailable())) {
9215 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9216 }
9217 return Register();
9218}
9219
9220Register fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9221 if (RetVT.SimpleTy != MVT::v16i8)
9222 return Register();
9223 if ((Subtarget->isNeonAvailable())) {
9224 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9225 }
9226 return Register();
9227}
9228
9229Register fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9230 if (RetVT.SimpleTy != MVT::v4i16)
9231 return Register();
9232 if ((Subtarget->isNeonAvailable())) {
9233 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9234 }
9235 return Register();
9236}
9237
9238Register fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9239 if (RetVT.SimpleTy != MVT::v8i16)
9240 return Register();
9241 if ((Subtarget->isNeonAvailable())) {
9242 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9243 }
9244 return Register();
9245}
9246
9247Register fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9248 if (RetVT.SimpleTy != MVT::v2i32)
9249 return Register();
9250 if ((Subtarget->isNeonAvailable())) {
9251 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9252 }
9253 return Register();
9254}
9255
9256Register fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9257 if (RetVT.SimpleTy != MVT::v4i32)
9258 return Register();
9259 if ((Subtarget->isNeonAvailable())) {
9260 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9261 }
9262 return Register();
9263}
9264
9265Register fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9266 if (RetVT.SimpleTy != MVT::v2i64)
9267 return Register();
9268 if ((Subtarget->isNeonAvailable())) {
9269 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9270 }
9271 return Register();
9272}
9273
9274Register fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9275 if (RetVT.SimpleTy != MVT::v4f16)
9276 return Register();
9277 if ((Subtarget->isNeonAvailable())) {
9278 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9279 }
9280 return Register();
9281}
9282
9283Register fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9284 if (RetVT.SimpleTy != MVT::v8f16)
9285 return Register();
9286 if ((Subtarget->isNeonAvailable())) {
9287 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9288 }
9289 return Register();
9290}
9291
9292Register fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9293 if (RetVT.SimpleTy != MVT::v4bf16)
9294 return Register();
9295 if ((Subtarget->isNeonAvailable())) {
9296 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9297 }
9298 return Register();
9299}
9300
9301Register fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9302 if (RetVT.SimpleTy != MVT::v8bf16)
9303 return Register();
9304 if ((Subtarget->isNeonAvailable())) {
9305 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9306 }
9307 return Register();
9308}
9309
9310Register fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9311 if (RetVT.SimpleTy != MVT::v2f32)
9312 return Register();
9313 if ((Subtarget->isNeonAvailable())) {
9314 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9315 }
9316 return Register();
9317}
9318
9319Register fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9320 if (RetVT.SimpleTy != MVT::v4f32)
9321 return Register();
9322 if ((Subtarget->isNeonAvailable())) {
9323 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9324 }
9325 return Register();
9326}
9327
9328Register fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9329 if (RetVT.SimpleTy != MVT::v2f64)
9330 return Register();
9331 if ((Subtarget->isNeonAvailable())) {
9332 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9333 }
9334 return Register();
9335}
9336
9337Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9338 if (RetVT.SimpleTy != MVT::nxv2i1)
9339 return Register();
9340 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9341 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9342 }
9343 return Register();
9344}
9345
9346Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9347 if (RetVT.SimpleTy != MVT::nxv4i1)
9348 return Register();
9349 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9350 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9351 }
9352 return Register();
9353}
9354
9355Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9356 if (RetVT.SimpleTy != MVT::nxv8i1)
9357 return Register();
9358 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9359 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9360 }
9361 return Register();
9362}
9363
9364Register fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9365 if (RetVT.SimpleTy != MVT::nxv16i1)
9366 return Register();
9367 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9368 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9369 }
9370 return Register();
9371}
9372
9373Register fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9374 if (RetVT.SimpleTy != MVT::nxv16i8)
9375 return Register();
9376 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9377 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9378 }
9379 return Register();
9380}
9381
9382Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9383 if (RetVT.SimpleTy != MVT::nxv8i16)
9384 return Register();
9385 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9386 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9387 }
9388 return Register();
9389}
9390
9391Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9392 if (RetVT.SimpleTy != MVT::nxv4i32)
9393 return Register();
9394 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9395 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9396 }
9397 return Register();
9398}
9399
9400Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9401 if (RetVT.SimpleTy != MVT::nxv2i64)
9402 return Register();
9403 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9404 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9405 }
9406 return Register();
9407}
9408
9409Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9410 if (RetVT.SimpleTy != MVT::nxv2f16)
9411 return Register();
9412 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9413 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9414 }
9415 return Register();
9416}
9417
9418Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9419 if (RetVT.SimpleTy != MVT::nxv4f16)
9420 return Register();
9421 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9422 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9423 }
9424 return Register();
9425}
9426
9427Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9428 if (RetVT.SimpleTy != MVT::nxv8f16)
9429 return Register();
9430 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9431 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9432 }
9433 return Register();
9434}
9435
9436Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9437 if (RetVT.SimpleTy != MVT::nxv2bf16)
9438 return Register();
9439 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9440 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9441 }
9442 return Register();
9443}
9444
9445Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9446 if (RetVT.SimpleTy != MVT::nxv4bf16)
9447 return Register();
9448 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9449 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9450 }
9451 return Register();
9452}
9453
9454Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9455 if (RetVT.SimpleTy != MVT::nxv8bf16)
9456 return Register();
9457 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9458 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9459 }
9460 return Register();
9461}
9462
9463Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9464 if (RetVT.SimpleTy != MVT::nxv2f32)
9465 return Register();
9466 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9467 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9468 }
9469 return Register();
9470}
9471
9472Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9473 if (RetVT.SimpleTy != MVT::nxv4f32)
9474 return Register();
9475 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9476 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9477 }
9478 return Register();
9479}
9480
9481Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9482 if (RetVT.SimpleTy != MVT::nxv2f64)
9483 return Register();
9484 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9485 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9486 }
9487 return Register();
9488}
9489
9490Register fastEmit_AArch64ISD_ZIP1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9491 switch (VT.SimpleTy) {
9492 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(RetVT, Op0, Op1);
9493 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(RetVT, Op0, Op1);
9494 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(RetVT, Op0, Op1);
9495 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(RetVT, Op0, Op1);
9496 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(RetVT, Op0, Op1);
9497 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(RetVT, Op0, Op1);
9498 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(RetVT, Op0, Op1);
9499 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(RetVT, Op0, Op1);
9500 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(RetVT, Op0, Op1);
9501 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(RetVT, Op0, Op1);
9502 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(RetVT, Op0, Op1);
9503 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(RetVT, Op0, Op1);
9504 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(RetVT, Op0, Op1);
9505 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(RetVT, Op0, Op1);
9506 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9507 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9508 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9509 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9510 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9511 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9512 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9513 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9514 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9515 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9516 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9517 case MVT::nxv2bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9518 case MVT::nxv4bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9519 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9520 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9521 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9522 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9523 default: return Register();
9524 }
9525}
9526
9527// FastEmit functions for AArch64ISD::ZIP2.
9528
9529Register fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9530 if (RetVT.SimpleTy != MVT::v8i8)
9531 return Register();
9532 if ((Subtarget->isNeonAvailable())) {
9533 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9534 }
9535 return Register();
9536}
9537
9538Register fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9539 if (RetVT.SimpleTy != MVT::v16i8)
9540 return Register();
9541 if ((Subtarget->isNeonAvailable())) {
9542 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9543 }
9544 return Register();
9545}
9546
9547Register fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9548 if (RetVT.SimpleTy != MVT::v4i16)
9549 return Register();
9550 if ((Subtarget->isNeonAvailable())) {
9551 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9552 }
9553 return Register();
9554}
9555
9556Register fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9557 if (RetVT.SimpleTy != MVT::v8i16)
9558 return Register();
9559 if ((Subtarget->isNeonAvailable())) {
9560 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9561 }
9562 return Register();
9563}
9564
9565Register fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9566 if (RetVT.SimpleTy != MVT::v2i32)
9567 return Register();
9568 if ((Subtarget->isNeonAvailable())) {
9569 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9570 }
9571 return Register();
9572}
9573
9574Register fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9575 if (RetVT.SimpleTy != MVT::v4i32)
9576 return Register();
9577 if ((Subtarget->isNeonAvailable())) {
9578 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9579 }
9580 return Register();
9581}
9582
9583Register fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9584 if (RetVT.SimpleTy != MVT::v2i64)
9585 return Register();
9586 if ((Subtarget->isNeonAvailable())) {
9587 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9588 }
9589 return Register();
9590}
9591
9592Register fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9593 if (RetVT.SimpleTy != MVT::v4f16)
9594 return Register();
9595 if ((Subtarget->isNeonAvailable())) {
9596 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9597 }
9598 return Register();
9599}
9600
9601Register fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9602 if (RetVT.SimpleTy != MVT::v8f16)
9603 return Register();
9604 if ((Subtarget->isNeonAvailable())) {
9605 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9606 }
9607 return Register();
9608}
9609
9610Register fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9611 if (RetVT.SimpleTy != MVT::v4bf16)
9612 return Register();
9613 if ((Subtarget->isNeonAvailable())) {
9614 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9615 }
9616 return Register();
9617}
9618
9619Register fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9620 if (RetVT.SimpleTy != MVT::v8bf16)
9621 return Register();
9622 if ((Subtarget->isNeonAvailable())) {
9623 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9624 }
9625 return Register();
9626}
9627
9628Register fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9629 if (RetVT.SimpleTy != MVT::v2f32)
9630 return Register();
9631 if ((Subtarget->isNeonAvailable())) {
9632 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9633 }
9634 return Register();
9635}
9636
9637Register fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9638 if (RetVT.SimpleTy != MVT::v4f32)
9639 return Register();
9640 if ((Subtarget->isNeonAvailable())) {
9641 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9642 }
9643 return Register();
9644}
9645
9646Register fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9647 if (RetVT.SimpleTy != MVT::v2f64)
9648 return Register();
9649 if ((Subtarget->isNeonAvailable())) {
9650 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9651 }
9652 return Register();
9653}
9654
9655Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9656 if (RetVT.SimpleTy != MVT::nxv2i1)
9657 return Register();
9658 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9659 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9660 }
9661 return Register();
9662}
9663
9664Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9665 if (RetVT.SimpleTy != MVT::nxv4i1)
9666 return Register();
9667 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9668 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9669 }
9670 return Register();
9671}
9672
9673Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9674 if (RetVT.SimpleTy != MVT::nxv8i1)
9675 return Register();
9676 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9677 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9678 }
9679 return Register();
9680}
9681
9682Register fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9683 if (RetVT.SimpleTy != MVT::nxv16i1)
9684 return Register();
9685 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9686 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9687 }
9688 return Register();
9689}
9690
9691Register fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9692 if (RetVT.SimpleTy != MVT::nxv16i8)
9693 return Register();
9694 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9695 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9696 }
9697 return Register();
9698}
9699
9700Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9701 if (RetVT.SimpleTy != MVT::nxv8i16)
9702 return Register();
9703 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9704 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9705 }
9706 return Register();
9707}
9708
9709Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9710 if (RetVT.SimpleTy != MVT::nxv4i32)
9711 return Register();
9712 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9713 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9714 }
9715 return Register();
9716}
9717
9718Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9719 if (RetVT.SimpleTy != MVT::nxv2i64)
9720 return Register();
9721 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9722 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9723 }
9724 return Register();
9725}
9726
9727Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9728 if (RetVT.SimpleTy != MVT::nxv2f16)
9729 return Register();
9730 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9731 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9732 }
9733 return Register();
9734}
9735
9736Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9737 if (RetVT.SimpleTy != MVT::nxv4f16)
9738 return Register();
9739 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9740 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9741 }
9742 return Register();
9743}
9744
9745Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9746 if (RetVT.SimpleTy != MVT::nxv8f16)
9747 return Register();
9748 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9749 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9750 }
9751 return Register();
9752}
9753
9754Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9755 if (RetVT.SimpleTy != MVT::nxv2bf16)
9756 return Register();
9757 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9758 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9759 }
9760 return Register();
9761}
9762
9763Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9764 if (RetVT.SimpleTy != MVT::nxv4bf16)
9765 return Register();
9766 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9767 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9768 }
9769 return Register();
9770}
9771
9772Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9773 if (RetVT.SimpleTy != MVT::nxv8bf16)
9774 return Register();
9775 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9776 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9777 }
9778 return Register();
9779}
9780
9781Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9782 if (RetVT.SimpleTy != MVT::nxv2f32)
9783 return Register();
9784 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9785 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9786 }
9787 return Register();
9788}
9789
9790Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9791 if (RetVT.SimpleTy != MVT::nxv4f32)
9792 return Register();
9793 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9794 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9795 }
9796 return Register();
9797}
9798
9799Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9800 if (RetVT.SimpleTy != MVT::nxv2f64)
9801 return Register();
9802 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9803 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9804 }
9805 return Register();
9806}
9807
9808Register fastEmit_AArch64ISD_ZIP2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9809 switch (VT.SimpleTy) {
9810 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(RetVT, Op0, Op1);
9811 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(RetVT, Op0, Op1);
9812 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(RetVT, Op0, Op1);
9813 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(RetVT, Op0, Op1);
9814 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(RetVT, Op0, Op1);
9815 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(RetVT, Op0, Op1);
9816 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(RetVT, Op0, Op1);
9817 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(RetVT, Op0, Op1);
9818 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(RetVT, Op0, Op1);
9819 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(RetVT, Op0, Op1);
9820 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(RetVT, Op0, Op1);
9821 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(RetVT, Op0, Op1);
9822 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(RetVT, Op0, Op1);
9823 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(RetVT, Op0, Op1);
9824 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9825 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9826 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9827 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9828 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9829 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9830 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9831 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9832 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9833 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9834 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9835 case MVT::nxv2bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9836 case MVT::nxv4bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9837 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9838 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9839 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9840 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9841 default: return Register();
9842 }
9843}
9844
9845// FastEmit functions for ISD::ABDS.
9846
9847Register fastEmit_ISD_ABDS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9848 if (RetVT.SimpleTy != MVT::v8i8)
9849 return Register();
9850 if ((Subtarget->isNeonAvailable())) {
9851 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9852 }
9853 return Register();
9854}
9855
9856Register fastEmit_ISD_ABDS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9857 if (RetVT.SimpleTy != MVT::v16i8)
9858 return Register();
9859 if ((Subtarget->isNeonAvailable())) {
9860 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9861 }
9862 return Register();
9863}
9864
9865Register fastEmit_ISD_ABDS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9866 if (RetVT.SimpleTy != MVT::v4i16)
9867 return Register();
9868 if ((Subtarget->isNeonAvailable())) {
9869 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9870 }
9871 return Register();
9872}
9873
9874Register fastEmit_ISD_ABDS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9875 if (RetVT.SimpleTy != MVT::v8i16)
9876 return Register();
9877 if ((Subtarget->isNeonAvailable())) {
9878 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9879 }
9880 return Register();
9881}
9882
9883Register fastEmit_ISD_ABDS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9884 if (RetVT.SimpleTy != MVT::v2i32)
9885 return Register();
9886 if ((Subtarget->isNeonAvailable())) {
9887 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9888 }
9889 return Register();
9890}
9891
9892Register fastEmit_ISD_ABDS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9893 if (RetVT.SimpleTy != MVT::v4i32)
9894 return Register();
9895 if ((Subtarget->isNeonAvailable())) {
9896 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9897 }
9898 return Register();
9899}
9900
9901Register fastEmit_ISD_ABDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9902 switch (VT.SimpleTy) {
9903 case MVT::v8i8: return fastEmit_ISD_ABDS_MVT_v8i8_rr(RetVT, Op0, Op1);
9904 case MVT::v16i8: return fastEmit_ISD_ABDS_MVT_v16i8_rr(RetVT, Op0, Op1);
9905 case MVT::v4i16: return fastEmit_ISD_ABDS_MVT_v4i16_rr(RetVT, Op0, Op1);
9906 case MVT::v8i16: return fastEmit_ISD_ABDS_MVT_v8i16_rr(RetVT, Op0, Op1);
9907 case MVT::v2i32: return fastEmit_ISD_ABDS_MVT_v2i32_rr(RetVT, Op0, Op1);
9908 case MVT::v4i32: return fastEmit_ISD_ABDS_MVT_v4i32_rr(RetVT, Op0, Op1);
9909 default: return Register();
9910 }
9911}
9912
9913// FastEmit functions for ISD::ABDU.
9914
9915Register fastEmit_ISD_ABDU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9916 if (RetVT.SimpleTy != MVT::v8i8)
9917 return Register();
9918 if ((Subtarget->isNeonAvailable())) {
9919 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9920 }
9921 return Register();
9922}
9923
9924Register fastEmit_ISD_ABDU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9925 if (RetVT.SimpleTy != MVT::v16i8)
9926 return Register();
9927 if ((Subtarget->isNeonAvailable())) {
9928 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9929 }
9930 return Register();
9931}
9932
9933Register fastEmit_ISD_ABDU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9934 if (RetVT.SimpleTy != MVT::v4i16)
9935 return Register();
9936 if ((Subtarget->isNeonAvailable())) {
9937 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9938 }
9939 return Register();
9940}
9941
9942Register fastEmit_ISD_ABDU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9943 if (RetVT.SimpleTy != MVT::v8i16)
9944 return Register();
9945 if ((Subtarget->isNeonAvailable())) {
9946 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9947 }
9948 return Register();
9949}
9950
9951Register fastEmit_ISD_ABDU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9952 if (RetVT.SimpleTy != MVT::v2i32)
9953 return Register();
9954 if ((Subtarget->isNeonAvailable())) {
9955 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9956 }
9957 return Register();
9958}
9959
9960Register fastEmit_ISD_ABDU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9961 if (RetVT.SimpleTy != MVT::v4i32)
9962 return Register();
9963 if ((Subtarget->isNeonAvailable())) {
9964 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9965 }
9966 return Register();
9967}
9968
9969Register fastEmit_ISD_ABDU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9970 switch (VT.SimpleTy) {
9971 case MVT::v8i8: return fastEmit_ISD_ABDU_MVT_v8i8_rr(RetVT, Op0, Op1);
9972 case MVT::v16i8: return fastEmit_ISD_ABDU_MVT_v16i8_rr(RetVT, Op0, Op1);
9973 case MVT::v4i16: return fastEmit_ISD_ABDU_MVT_v4i16_rr(RetVT, Op0, Op1);
9974 case MVT::v8i16: return fastEmit_ISD_ABDU_MVT_v8i16_rr(RetVT, Op0, Op1);
9975 case MVT::v2i32: return fastEmit_ISD_ABDU_MVT_v2i32_rr(RetVT, Op0, Op1);
9976 case MVT::v4i32: return fastEmit_ISD_ABDU_MVT_v4i32_rr(RetVT, Op0, Op1);
9977 default: return Register();
9978 }
9979}
9980
9981// FastEmit functions for ISD::ADD.
9982
9983Register fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
9984 if (RetVT.SimpleTy != MVT::i32)
9985 return Register();
9986 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
9987}
9988
9989Register fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
9990 if (RetVT.SimpleTy != MVT::i64)
9991 return Register();
9992 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
9993}
9994
9995Register fastEmit_ISD_ADD_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9996 if (RetVT.SimpleTy != MVT::v8i8)
9997 return Register();
9998 if ((Subtarget->isNeonAvailable())) {
9999 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10000 }
10001 return Register();
10002}
10003
10004Register fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10005 if (RetVT.SimpleTy != MVT::v16i8)
10006 return Register();
10007 if ((Subtarget->isNeonAvailable())) {
10008 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10009 }
10010 return Register();
10011}
10012
10013Register fastEmit_ISD_ADD_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10014 if (RetVT.SimpleTy != MVT::v4i16)
10015 return Register();
10016 if ((Subtarget->isNeonAvailable())) {
10017 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10018 }
10019 return Register();
10020}
10021
10022Register fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10023 if (RetVT.SimpleTy != MVT::v8i16)
10024 return Register();
10025 if ((Subtarget->isNeonAvailable())) {
10026 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10027 }
10028 return Register();
10029}
10030
10031Register fastEmit_ISD_ADD_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10032 if (RetVT.SimpleTy != MVT::v2i32)
10033 return Register();
10034 if ((Subtarget->isNeonAvailable())) {
10035 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10036 }
10037 return Register();
10038}
10039
10040Register fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10041 if (RetVT.SimpleTy != MVT::v4i32)
10042 return Register();
10043 if ((Subtarget->isNeonAvailable())) {
10044 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10045 }
10046 return Register();
10047}
10048
10049Register fastEmit_ISD_ADD_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
10050 if (RetVT.SimpleTy != MVT::v1i64)
10051 return Register();
10052 if ((Subtarget->isNeonAvailable())) {
10053 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
10054 }
10055 return Register();
10056}
10057
10058Register fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10059 if (RetVT.SimpleTy != MVT::v2i64)
10060 return Register();
10061 if ((Subtarget->isNeonAvailable())) {
10062 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10063 }
10064 return Register();
10065}
10066
10067Register fastEmit_ISD_ADD_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10068 if (RetVT.SimpleTy != MVT::nxv16i8)
10069 return Register();
10070 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10071 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
10072 }
10073 return Register();
10074}
10075
10076Register fastEmit_ISD_ADD_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10077 if (RetVT.SimpleTy != MVT::nxv8i16)
10078 return Register();
10079 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10080 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10081 }
10082 return Register();
10083}
10084
10085Register fastEmit_ISD_ADD_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10086 if (RetVT.SimpleTy != MVT::nxv4i32)
10087 return Register();
10088 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10089 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10090 }
10091 return Register();
10092}
10093
10094Register fastEmit_ISD_ADD_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10095 if (RetVT.SimpleTy != MVT::nxv2i64)
10096 return Register();
10097 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10098 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10099 }
10100 return Register();
10101}
10102
10103Register fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10104 switch (VT.SimpleTy) {
10105 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
10106 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op1);
10107 case MVT::v8i8: return fastEmit_ISD_ADD_MVT_v8i8_rr(RetVT, Op0, Op1);
10108 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
10109 case MVT::v4i16: return fastEmit_ISD_ADD_MVT_v4i16_rr(RetVT, Op0, Op1);
10110 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
10111 case MVT::v2i32: return fastEmit_ISD_ADD_MVT_v2i32_rr(RetVT, Op0, Op1);
10112 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
10113 case MVT::v1i64: return fastEmit_ISD_ADD_MVT_v1i64_rr(RetVT, Op0, Op1);
10114 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
10115 case MVT::nxv16i8: return fastEmit_ISD_ADD_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10116 case MVT::nxv8i16: return fastEmit_ISD_ADD_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10117 case MVT::nxv4i32: return fastEmit_ISD_ADD_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10118 case MVT::nxv2i64: return fastEmit_ISD_ADD_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10119 default: return Register();
10120 }
10121}
10122
10123// FastEmit functions for ISD::AND.
10124
10125Register fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10126 if (RetVT.SimpleTy != MVT::i32)
10127 return Register();
10128 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
10129}
10130
10131Register fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10132 if (RetVT.SimpleTy != MVT::i64)
10133 return Register();
10134 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
10135}
10136
10137Register fastEmit_ISD_AND_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10138 if (RetVT.SimpleTy != MVT::v8i8)
10139 return Register();
10140 if ((Subtarget->isNeonAvailable())) {
10141 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10142 }
10143 return Register();
10144}
10145
10146Register fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10147 if (RetVT.SimpleTy != MVT::v16i8)
10148 return Register();
10149 if ((Subtarget->isNeonAvailable())) {
10150 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10151 }
10152 return Register();
10153}
10154
10155Register fastEmit_ISD_AND_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10156 if (RetVT.SimpleTy != MVT::v4i16)
10157 return Register();
10158 if ((Subtarget->isNeonAvailable())) {
10159 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10160 }
10161 return Register();
10162}
10163
10164Register fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10165 if (RetVT.SimpleTy != MVT::v8i16)
10166 return Register();
10167 if ((Subtarget->isNeonAvailable())) {
10168 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10169 }
10170 return Register();
10171}
10172
10173Register fastEmit_ISD_AND_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10174 if (RetVT.SimpleTy != MVT::v2i32)
10175 return Register();
10176 if ((Subtarget->isNeonAvailable())) {
10177 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10178 }
10179 return Register();
10180}
10181
10182Register fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10183 if (RetVT.SimpleTy != MVT::v4i32)
10184 return Register();
10185 if ((Subtarget->isNeonAvailable())) {
10186 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10187 }
10188 return Register();
10189}
10190
10191Register fastEmit_ISD_AND_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
10192 if (RetVT.SimpleTy != MVT::v1i64)
10193 return Register();
10194 if ((Subtarget->isNeonAvailable())) {
10195 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10196 }
10197 return Register();
10198}
10199
10200Register fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10201 if (RetVT.SimpleTy != MVT::v2i64)
10202 return Register();
10203 if ((Subtarget->isNeonAvailable())) {
10204 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10205 }
10206 return Register();
10207}
10208
10209Register fastEmit_ISD_AND_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10210 if (RetVT.SimpleTy != MVT::nxv16i8)
10211 return Register();
10212 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10213 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10214 }
10215 return Register();
10216}
10217
10218Register fastEmit_ISD_AND_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10219 if (RetVT.SimpleTy != MVT::nxv8i16)
10220 return Register();
10221 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10222 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10223 }
10224 return Register();
10225}
10226
10227Register fastEmit_ISD_AND_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10228 if (RetVT.SimpleTy != MVT::nxv4i32)
10229 return Register();
10230 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10231 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10232 }
10233 return Register();
10234}
10235
10236Register fastEmit_ISD_AND_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10237 if (RetVT.SimpleTy != MVT::nxv2i64)
10238 return Register();
10239 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10240 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10241 }
10242 return Register();
10243}
10244
10245Register fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10246 switch (VT.SimpleTy) {
10247 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
10248 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op1);
10249 case MVT::v8i8: return fastEmit_ISD_AND_MVT_v8i8_rr(RetVT, Op0, Op1);
10250 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
10251 case MVT::v4i16: return fastEmit_ISD_AND_MVT_v4i16_rr(RetVT, Op0, Op1);
10252 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
10253 case MVT::v2i32: return fastEmit_ISD_AND_MVT_v2i32_rr(RetVT, Op0, Op1);
10254 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
10255 case MVT::v1i64: return fastEmit_ISD_AND_MVT_v1i64_rr(RetVT, Op0, Op1);
10256 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
10257 case MVT::nxv16i8: return fastEmit_ISD_AND_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10258 case MVT::nxv8i16: return fastEmit_ISD_AND_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10259 case MVT::nxv4i32: return fastEmit_ISD_AND_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10260 case MVT::nxv2i64: return fastEmit_ISD_AND_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10261 default: return Register();
10262 }
10263}
10264
10265// FastEmit functions for ISD::AVGCEILS.
10266
10267Register fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10268 if (RetVT.SimpleTy != MVT::v8i8)
10269 return Register();
10270 if ((Subtarget->isNeonAvailable())) {
10271 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10272 }
10273 return Register();
10274}
10275
10276Register fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10277 if (RetVT.SimpleTy != MVT::v16i8)
10278 return Register();
10279 if ((Subtarget->isNeonAvailable())) {
10280 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10281 }
10282 return Register();
10283}
10284
10285Register fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10286 if (RetVT.SimpleTy != MVT::v4i16)
10287 return Register();
10288 if ((Subtarget->isNeonAvailable())) {
10289 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10290 }
10291 return Register();
10292}
10293
10294Register fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10295 if (RetVT.SimpleTy != MVT::v8i16)
10296 return Register();
10297 if ((Subtarget->isNeonAvailable())) {
10298 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10299 }
10300 return Register();
10301}
10302
10303Register fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10304 if (RetVT.SimpleTy != MVT::v2i32)
10305 return Register();
10306 if ((Subtarget->isNeonAvailable())) {
10307 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10308 }
10309 return Register();
10310}
10311
10312Register fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10313 if (RetVT.SimpleTy != MVT::v4i32)
10314 return Register();
10315 if ((Subtarget->isNeonAvailable())) {
10316 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10317 }
10318 return Register();
10319}
10320
10321Register fastEmit_ISD_AVGCEILS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10322 switch (VT.SimpleTy) {
10323 case MVT::v8i8: return fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(RetVT, Op0, Op1);
10324 case MVT::v16i8: return fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(RetVT, Op0, Op1);
10325 case MVT::v4i16: return fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(RetVT, Op0, Op1);
10326 case MVT::v8i16: return fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(RetVT, Op0, Op1);
10327 case MVT::v2i32: return fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(RetVT, Op0, Op1);
10328 case MVT::v4i32: return fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(RetVT, Op0, Op1);
10329 default: return Register();
10330 }
10331}
10332
10333// FastEmit functions for ISD::AVGCEILU.
10334
10335Register fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10336 if (RetVT.SimpleTy != MVT::v8i8)
10337 return Register();
10338 if ((Subtarget->isNeonAvailable())) {
10339 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10340 }
10341 return Register();
10342}
10343
10344Register fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10345 if (RetVT.SimpleTy != MVT::v16i8)
10346 return Register();
10347 if ((Subtarget->isNeonAvailable())) {
10348 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10349 }
10350 return Register();
10351}
10352
10353Register fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10354 if (RetVT.SimpleTy != MVT::v4i16)
10355 return Register();
10356 if ((Subtarget->isNeonAvailable())) {
10357 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10358 }
10359 return Register();
10360}
10361
10362Register fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10363 if (RetVT.SimpleTy != MVT::v8i16)
10364 return Register();
10365 if ((Subtarget->isNeonAvailable())) {
10366 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10367 }
10368 return Register();
10369}
10370
10371Register fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10372 if (RetVT.SimpleTy != MVT::v2i32)
10373 return Register();
10374 if ((Subtarget->isNeonAvailable())) {
10375 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10376 }
10377 return Register();
10378}
10379
10380Register fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10381 if (RetVT.SimpleTy != MVT::v4i32)
10382 return Register();
10383 if ((Subtarget->isNeonAvailable())) {
10384 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10385 }
10386 return Register();
10387}
10388
10389Register fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10390 switch (VT.SimpleTy) {
10391 case MVT::v8i8: return fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(RetVT, Op0, Op1);
10392 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
10393 case MVT::v4i16: return fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(RetVT, Op0, Op1);
10394 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
10395 case MVT::v2i32: return fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(RetVT, Op0, Op1);
10396 case MVT::v4i32: return fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(RetVT, Op0, Op1);
10397 default: return Register();
10398 }
10399}
10400
10401// FastEmit functions for ISD::AVGFLOORS.
10402
10403Register fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10404 if (RetVT.SimpleTy != MVT::v8i8)
10405 return Register();
10406 if ((Subtarget->isNeonAvailable())) {
10407 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10408 }
10409 return Register();
10410}
10411
10412Register fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10413 if (RetVT.SimpleTy != MVT::v16i8)
10414 return Register();
10415 if ((Subtarget->isNeonAvailable())) {
10416 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10417 }
10418 return Register();
10419}
10420
10421Register fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10422 if (RetVT.SimpleTy != MVT::v4i16)
10423 return Register();
10424 if ((Subtarget->isNeonAvailable())) {
10425 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10426 }
10427 return Register();
10428}
10429
10430Register fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10431 if (RetVT.SimpleTy != MVT::v8i16)
10432 return Register();
10433 if ((Subtarget->isNeonAvailable())) {
10434 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10435 }
10436 return Register();
10437}
10438
10439Register fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10440 if (RetVT.SimpleTy != MVT::v2i32)
10441 return Register();
10442 if ((Subtarget->isNeonAvailable())) {
10443 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10444 }
10445 return Register();
10446}
10447
10448Register fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10449 if (RetVT.SimpleTy != MVT::v4i32)
10450 return Register();
10451 if ((Subtarget->isNeonAvailable())) {
10452 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10453 }
10454 return Register();
10455}
10456
10457Register fastEmit_ISD_AVGFLOORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10458 switch (VT.SimpleTy) {
10459 case MVT::v8i8: return fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(RetVT, Op0, Op1);
10460 case MVT::v16i8: return fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(RetVT, Op0, Op1);
10461 case MVT::v4i16: return fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(RetVT, Op0, Op1);
10462 case MVT::v8i16: return fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(RetVT, Op0, Op1);
10463 case MVT::v2i32: return fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(RetVT, Op0, Op1);
10464 case MVT::v4i32: return fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(RetVT, Op0, Op1);
10465 default: return Register();
10466 }
10467}
10468
10469// FastEmit functions for ISD::AVGFLOORU.
10470
10471Register fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10472 if (RetVT.SimpleTy != MVT::v8i8)
10473 return Register();
10474 if ((Subtarget->isNeonAvailable())) {
10475 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10476 }
10477 return Register();
10478}
10479
10480Register fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10481 if (RetVT.SimpleTy != MVT::v16i8)
10482 return Register();
10483 if ((Subtarget->isNeonAvailable())) {
10484 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10485 }
10486 return Register();
10487}
10488
10489Register fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10490 if (RetVT.SimpleTy != MVT::v4i16)
10491 return Register();
10492 if ((Subtarget->isNeonAvailable())) {
10493 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10494 }
10495 return Register();
10496}
10497
10498Register fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10499 if (RetVT.SimpleTy != MVT::v8i16)
10500 return Register();
10501 if ((Subtarget->isNeonAvailable())) {
10502 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10503 }
10504 return Register();
10505}
10506
10507Register fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10508 if (RetVT.SimpleTy != MVT::v2i32)
10509 return Register();
10510 if ((Subtarget->isNeonAvailable())) {
10511 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10512 }
10513 return Register();
10514}
10515
10516Register fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10517 if (RetVT.SimpleTy != MVT::v4i32)
10518 return Register();
10519 if ((Subtarget->isNeonAvailable())) {
10520 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10521 }
10522 return Register();
10523}
10524
10525Register fastEmit_ISD_AVGFLOORU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10526 switch (VT.SimpleTy) {
10527 case MVT::v8i8: return fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(RetVT, Op0, Op1);
10528 case MVT::v16i8: return fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(RetVT, Op0, Op1);
10529 case MVT::v4i16: return fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(RetVT, Op0, Op1);
10530 case MVT::v8i16: return fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(RetVT, Op0, Op1);
10531 case MVT::v2i32: return fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(RetVT, Op0, Op1);
10532 case MVT::v4i32: return fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(RetVT, Op0, Op1);
10533 default: return Register();
10534 }
10535}
10536
10537// FastEmit functions for ISD::CLMUL.
10538
10539Register fastEmit_ISD_CLMUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10540 if (RetVT.SimpleTy != MVT::v8i8)
10541 return Register();
10542 if ((Subtarget->isNeonAvailable())) {
10543 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10544 }
10545 return Register();
10546}
10547
10548Register fastEmit_ISD_CLMUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10549 if (RetVT.SimpleTy != MVT::v16i8)
10550 return Register();
10551 if ((Subtarget->isNeonAvailable())) {
10552 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10553 }
10554 return Register();
10555}
10556
10557Register fastEmit_ISD_CLMUL_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10558 if (RetVT.SimpleTy != MVT::nxv16i8)
10559 return Register();
10560 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2() || Subtarget->hasSME()))) {
10561 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMUL_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
10562 }
10563 return Register();
10564}
10565
10566Register fastEmit_ISD_CLMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10567 switch (VT.SimpleTy) {
10568 case MVT::v8i8: return fastEmit_ISD_CLMUL_MVT_v8i8_rr(RetVT, Op0, Op1);
10569 case MVT::v16i8: return fastEmit_ISD_CLMUL_MVT_v16i8_rr(RetVT, Op0, Op1);
10570 case MVT::nxv16i8: return fastEmit_ISD_CLMUL_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10571 default: return Register();
10572 }
10573}
10574
10575// FastEmit functions for ISD::CONCAT_VECTORS.
10576
10577Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(MVT RetVT, Register Op0, Register Op1) {
10578 if (RetVT.SimpleTy != MVT::nxv2i1)
10579 return Register();
10580 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10581 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
10582 }
10583 return Register();
10584}
10585
10586Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
10587 if (RetVT.SimpleTy != MVT::nxv4i1)
10588 return Register();
10589 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10590 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
10591 }
10592 return Register();
10593}
10594
10595Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
10596 if (RetVT.SimpleTy != MVT::nxv8i1)
10597 return Register();
10598 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10599 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
10600 }
10601 return Register();
10602}
10603
10604Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
10605 if (RetVT.SimpleTy != MVT::nxv16i1)
10606 return Register();
10607 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10608 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
10609 }
10610 return Register();
10611}
10612
10613Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
10614 if (RetVT.SimpleTy != MVT::nxv4f16)
10615 return Register();
10616 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10617 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10618 }
10619 return Register();
10620}
10621
10622Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10623 if (RetVT.SimpleTy != MVT::nxv8f16)
10624 return Register();
10625 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10626 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10627 }
10628 return Register();
10629}
10630
10631Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10632 if (RetVT.SimpleTy != MVT::nxv4bf16)
10633 return Register();
10634 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10635 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10636 }
10637 return Register();
10638}
10639
10640Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10641 if (RetVT.SimpleTy != MVT::nxv8bf16)
10642 return Register();
10643 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10644 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10645 }
10646 return Register();
10647}
10648
10649Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10650 if (RetVT.SimpleTy != MVT::nxv4f32)
10651 return Register();
10652 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10653 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10654 }
10655 return Register();
10656}
10657
10658Register fastEmit_ISD_CONCAT_VECTORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10659 switch (VT.SimpleTy) {
10660 case MVT::nxv1i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(RetVT, Op0, Op1);
10661 case MVT::nxv2i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(RetVT, Op0, Op1);
10662 case MVT::nxv4i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(RetVT, Op0, Op1);
10663 case MVT::nxv8i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(RetVT, Op0, Op1);
10664 case MVT::nxv2f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(RetVT, Op0, Op1);
10665 case MVT::nxv4f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(RetVT, Op0, Op1);
10666 case MVT::nxv2bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
10667 case MVT::nxv4bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
10668 case MVT::nxv2f32: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(RetVT, Op0, Op1);
10669 default: return Register();
10670 }
10671}
10672
10673// FastEmit functions for ISD::FADD.
10674
10675Register fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10676 if (RetVT.SimpleTy != MVT::f16)
10677 return Register();
10678 if ((Subtarget->hasFullFP16())) {
10679 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
10680 }
10681 return Register();
10682}
10683
10684Register fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10685 if (RetVT.SimpleTy != MVT::f32)
10686 return Register();
10687 if ((Subtarget->hasFPARMv8())) {
10688 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
10689 }
10690 return Register();
10691}
10692
10693Register fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
10694 if (RetVT.SimpleTy != MVT::f64)
10695 return Register();
10696 if ((Subtarget->hasFPARMv8())) {
10697 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10698 }
10699 return Register();
10700}
10701
10702Register fastEmit_ISD_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10703 if (RetVT.SimpleTy != MVT::v4f16)
10704 return Register();
10705 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10706 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10707 }
10708 return Register();
10709}
10710
10711Register fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10712 if (RetVT.SimpleTy != MVT::v8f16)
10713 return Register();
10714 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10715 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10716 }
10717 return Register();
10718}
10719
10720Register fastEmit_ISD_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10721 if (RetVT.SimpleTy != MVT::v2f32)
10722 return Register();
10723 if ((Subtarget->isNeonAvailable())) {
10724 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10725 }
10726 return Register();
10727}
10728
10729Register fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10730 if (RetVT.SimpleTy != MVT::v4f32)
10731 return Register();
10732 if ((Subtarget->isNeonAvailable())) {
10733 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10734 }
10735 return Register();
10736}
10737
10738Register fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10739 if (RetVT.SimpleTy != MVT::v2f64)
10740 return Register();
10741 if ((Subtarget->isNeonAvailable())) {
10742 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10743 }
10744 return Register();
10745}
10746
10747Register fastEmit_ISD_FADD_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10748 if (RetVT.SimpleTy != MVT::nxv8f16)
10749 return Register();
10750 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10751 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10752 }
10753 return Register();
10754}
10755
10756Register fastEmit_ISD_FADD_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10757 if (RetVT.SimpleTy != MVT::nxv8bf16)
10758 return Register();
10759 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
10760 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFADD_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10761 }
10762 return Register();
10763}
10764
10765Register fastEmit_ISD_FADD_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10766 if (RetVT.SimpleTy != MVT::nxv4f32)
10767 return Register();
10768 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10769 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10770 }
10771 return Register();
10772}
10773
10774Register fastEmit_ISD_FADD_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10775 if (RetVT.SimpleTy != MVT::nxv2f64)
10776 return Register();
10777 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10778 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10779 }
10780 return Register();
10781}
10782
10783Register fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10784 switch (VT.SimpleTy) {
10785 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
10786 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
10787 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
10788 case MVT::v4f16: return fastEmit_ISD_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
10789 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
10790 case MVT::v2f32: return fastEmit_ISD_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
10791 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
10792 case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
10793 case MVT::nxv8f16: return fastEmit_ISD_FADD_MVT_nxv8f16_rr(RetVT, Op0, Op1);
10794 case MVT::nxv8bf16: return fastEmit_ISD_FADD_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
10795 case MVT::nxv4f32: return fastEmit_ISD_FADD_MVT_nxv4f32_rr(RetVT, Op0, Op1);
10796 case MVT::nxv2f64: return fastEmit_ISD_FADD_MVT_nxv2f64_rr(RetVT, Op0, Op1);
10797 default: return Register();
10798 }
10799}
10800
10801// FastEmit functions for ISD::FDIV.
10802
10803Register fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10804 if (RetVT.SimpleTy != MVT::f16)
10805 return Register();
10806 if ((Subtarget->hasFullFP16())) {
10807 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
10808 }
10809 return Register();
10810}
10811
10812Register fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10813 if (RetVT.SimpleTy != MVT::f32)
10814 return Register();
10815 if ((Subtarget->hasFPARMv8())) {
10816 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
10817 }
10818 return Register();
10819}
10820
10821Register fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
10822 if (RetVT.SimpleTy != MVT::f64)
10823 return Register();
10824 if ((Subtarget->hasFPARMv8())) {
10825 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10826 }
10827 return Register();
10828}
10829
10830Register fastEmit_ISD_FDIV_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10831 if (RetVT.SimpleTy != MVT::v4f16)
10832 return Register();
10833 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10834 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10835 }
10836 return Register();
10837}
10838
10839Register fastEmit_ISD_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10840 if (RetVT.SimpleTy != MVT::v8f16)
10841 return Register();
10842 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10843 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10844 }
10845 return Register();
10846}
10847
10848Register fastEmit_ISD_FDIV_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10849 if (RetVT.SimpleTy != MVT::v2f32)
10850 return Register();
10851 if ((Subtarget->isNeonAvailable())) {
10852 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10853 }
10854 return Register();
10855}
10856
10857Register fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10858 if (RetVT.SimpleTy != MVT::v4f32)
10859 return Register();
10860 if ((Subtarget->isNeonAvailable())) {
10861 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10862 }
10863 return Register();
10864}
10865
10866Register fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10867 if (RetVT.SimpleTy != MVT::v2f64)
10868 return Register();
10869 if ((Subtarget->isNeonAvailable())) {
10870 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10871 }
10872 return Register();
10873}
10874
10875Register fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10876 switch (VT.SimpleTy) {
10877 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
10878 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
10879 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
10880 case MVT::v4f16: return fastEmit_ISD_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1);
10881 case MVT::v8f16: return fastEmit_ISD_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
10882 case MVT::v2f32: return fastEmit_ISD_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1);
10883 case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
10884 case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
10885 default: return Register();
10886 }
10887}
10888
10889// FastEmit functions for ISD::FMAXIMUM.
10890
10891Register fastEmit_ISD_FMAXIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10892 if (RetVT.SimpleTy != MVT::f16)
10893 return Register();
10894 if ((Subtarget->hasFullFP16())) {
10895 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
10896 }
10897 return Register();
10898}
10899
10900Register fastEmit_ISD_FMAXIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10901 if (RetVT.SimpleTy != MVT::f32)
10902 return Register();
10903 if ((Subtarget->hasFPARMv8())) {
10904 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
10905 }
10906 return Register();
10907}
10908
10909Register fastEmit_ISD_FMAXIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
10910 if (RetVT.SimpleTy != MVT::f64)
10911 return Register();
10912 if ((Subtarget->hasFPARMv8())) {
10913 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10914 }
10915 return Register();
10916}
10917
10918Register fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10919 if (RetVT.SimpleTy != MVT::v4f16)
10920 return Register();
10921 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10922 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10923 }
10924 return Register();
10925}
10926
10927Register fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10928 if (RetVT.SimpleTy != MVT::v8f16)
10929 return Register();
10930 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10931 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10932 }
10933 return Register();
10934}
10935
10936Register fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10937 if (RetVT.SimpleTy != MVT::v2f32)
10938 return Register();
10939 if ((Subtarget->isNeonAvailable())) {
10940 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10941 }
10942 return Register();
10943}
10944
10945Register fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10946 if (RetVT.SimpleTy != MVT::v4f32)
10947 return Register();
10948 if ((Subtarget->isNeonAvailable())) {
10949 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10950 }
10951 return Register();
10952}
10953
10954Register fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
10955 if (RetVT.SimpleTy != MVT::v1f64)
10956 return Register();
10957 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10958}
10959
10960Register fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10961 if (RetVT.SimpleTy != MVT::v2f64)
10962 return Register();
10963 if ((Subtarget->isNeonAvailable())) {
10964 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10965 }
10966 return Register();
10967}
10968
10969Register fastEmit_ISD_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10970 switch (VT.SimpleTy) {
10971 case MVT::f16: return fastEmit_ISD_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1);
10972 case MVT::f32: return fastEmit_ISD_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1);
10973 case MVT::f64: return fastEmit_ISD_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1);
10974 case MVT::v4f16: return fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
10975 case MVT::v8f16: return fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
10976 case MVT::v2f32: return fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
10977 case MVT::v4f32: return fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
10978 case MVT::v1f64: return fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(RetVT, Op0, Op1);
10979 case MVT::v2f64: return fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
10980 default: return Register();
10981 }
10982}
10983
10984// FastEmit functions for ISD::FMAXNUM.
10985
10986Register fastEmit_ISD_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10987 if (RetVT.SimpleTy != MVT::f16)
10988 return Register();
10989 if ((Subtarget->hasFullFP16())) {
10990 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
10991 }
10992 return Register();
10993}
10994
10995Register fastEmit_ISD_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10996 if (RetVT.SimpleTy != MVT::f32)
10997 return Register();
10998 if ((Subtarget->hasFPARMv8())) {
10999 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11000 }
11001 return Register();
11002}
11003
11004Register fastEmit_ISD_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11005 if (RetVT.SimpleTy != MVT::f64)
11006 return Register();
11007 if ((Subtarget->hasFPARMv8())) {
11008 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11009 }
11010 return Register();
11011}
11012
11013Register fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11014 if (RetVT.SimpleTy != MVT::v4f16)
11015 return Register();
11016 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11017 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11018 }
11019 return Register();
11020}
11021
11022Register fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11023 if (RetVT.SimpleTy != MVT::v8f16)
11024 return Register();
11025 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11026 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11027 }
11028 return Register();
11029}
11030
11031Register fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11032 if (RetVT.SimpleTy != MVT::v2f32)
11033 return Register();
11034 if ((Subtarget->isNeonAvailable())) {
11035 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11036 }
11037 return Register();
11038}
11039
11040Register fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11041 if (RetVT.SimpleTy != MVT::v4f32)
11042 return Register();
11043 if ((Subtarget->isNeonAvailable())) {
11044 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11045 }
11046 return Register();
11047}
11048
11049Register fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11050 if (RetVT.SimpleTy != MVT::v1f64)
11051 return Register();
11052 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11053}
11054
11055Register fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11056 if (RetVT.SimpleTy != MVT::v2f64)
11057 return Register();
11058 if ((Subtarget->isNeonAvailable())) {
11059 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11060 }
11061 return Register();
11062}
11063
11064Register fastEmit_ISD_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11065 switch (VT.SimpleTy) {
11066 case MVT::f16: return fastEmit_ISD_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
11067 case MVT::f32: return fastEmit_ISD_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
11068 case MVT::f64: return fastEmit_ISD_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
11069 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11070 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11071 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11072 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11073 case MVT::v1f64: return fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11074 case MVT::v2f64: return fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11075 default: return Register();
11076 }
11077}
11078
11079// FastEmit functions for ISD::FMAXNUM_IEEE.
11080
11081Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11082 if (RetVT.SimpleTy != MVT::f16)
11083 return Register();
11084 if ((Subtarget->hasFullFP16())) {
11085 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11086 }
11087 return Register();
11088}
11089
11090Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11091 if (RetVT.SimpleTy != MVT::f32)
11092 return Register();
11093 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11094}
11095
11096Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11097 if (RetVT.SimpleTy != MVT::f64)
11098 return Register();
11099 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11100}
11101
11102Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11103 if (RetVT.SimpleTy != MVT::v4f16)
11104 return Register();
11105 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11106 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11107 }
11108 return Register();
11109}
11110
11111Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11112 if (RetVT.SimpleTy != MVT::v8f16)
11113 return Register();
11114 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11115 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11116 }
11117 return Register();
11118}
11119
11120Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11121 if (RetVT.SimpleTy != MVT::v2f32)
11122 return Register();
11123 if ((Subtarget->isNeonAvailable())) {
11124 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11125 }
11126 return Register();
11127}
11128
11129Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11130 if (RetVT.SimpleTy != MVT::v4f32)
11131 return Register();
11132 if ((Subtarget->isNeonAvailable())) {
11133 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11134 }
11135 return Register();
11136}
11137
11138Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11139 if (RetVT.SimpleTy != MVT::v2f64)
11140 return Register();
11141 if ((Subtarget->isNeonAvailable())) {
11142 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11143 }
11144 return Register();
11145}
11146
11147Register fastEmit_ISD_FMAXNUM_IEEE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11148 switch (VT.SimpleTy) {
11149 case MVT::f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f16_rr(RetVT, Op0, Op1);
11150 case MVT::f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f32_rr(RetVT, Op0, Op1);
11151 case MVT::f64: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f64_rr(RetVT, Op0, Op1);
11152 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f16_rr(RetVT, Op0, Op1);
11153 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v8f16_rr(RetVT, Op0, Op1);
11154 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f32_rr(RetVT, Op0, Op1);
11155 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f32_rr(RetVT, Op0, Op1);
11156 case MVT::v2f64: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f64_rr(RetVT, Op0, Op1);
11157 default: return Register();
11158 }
11159}
11160
11161// FastEmit functions for ISD::FMINIMUM.
11162
11163Register fastEmit_ISD_FMINIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11164 if (RetVT.SimpleTy != MVT::f16)
11165 return Register();
11166 if ((Subtarget->hasFullFP16())) {
11167 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11168 }
11169 return Register();
11170}
11171
11172Register fastEmit_ISD_FMINIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11173 if (RetVT.SimpleTy != MVT::f32)
11174 return Register();
11175 if ((Subtarget->hasFPARMv8())) {
11176 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11177 }
11178 return Register();
11179}
11180
11181Register fastEmit_ISD_FMINIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11182 if (RetVT.SimpleTy != MVT::f64)
11183 return Register();
11184 if ((Subtarget->hasFPARMv8())) {
11185 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11186 }
11187 return Register();
11188}
11189
11190Register fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11191 if (RetVT.SimpleTy != MVT::v4f16)
11192 return Register();
11193 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11194 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11195 }
11196 return Register();
11197}
11198
11199Register fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11200 if (RetVT.SimpleTy != MVT::v8f16)
11201 return Register();
11202 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11203 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11204 }
11205 return Register();
11206}
11207
11208Register fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11209 if (RetVT.SimpleTy != MVT::v2f32)
11210 return Register();
11211 if ((Subtarget->isNeonAvailable())) {
11212 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11213 }
11214 return Register();
11215}
11216
11217Register fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11218 if (RetVT.SimpleTy != MVT::v4f32)
11219 return Register();
11220 if ((Subtarget->isNeonAvailable())) {
11221 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11222 }
11223 return Register();
11224}
11225
11226Register fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11227 if (RetVT.SimpleTy != MVT::v1f64)
11228 return Register();
11229 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11230}
11231
11232Register fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11233 if (RetVT.SimpleTy != MVT::v2f64)
11234 return Register();
11235 if ((Subtarget->isNeonAvailable())) {
11236 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11237 }
11238 return Register();
11239}
11240
11241Register fastEmit_ISD_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11242 switch (VT.SimpleTy) {
11243 case MVT::f16: return fastEmit_ISD_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1);
11244 case MVT::f32: return fastEmit_ISD_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1);
11245 case MVT::f64: return fastEmit_ISD_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1);
11246 case MVT::v4f16: return fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11247 case MVT::v8f16: return fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11248 case MVT::v2f32: return fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11249 case MVT::v4f32: return fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11250 case MVT::v1f64: return fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11251 case MVT::v2f64: return fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11252 default: return Register();
11253 }
11254}
11255
11256// FastEmit functions for ISD::FMINNUM.
11257
11258Register fastEmit_ISD_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11259 if (RetVT.SimpleTy != MVT::f16)
11260 return Register();
11261 if ((Subtarget->hasFullFP16())) {
11262 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11263 }
11264 return Register();
11265}
11266
11267Register fastEmit_ISD_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11268 if (RetVT.SimpleTy != MVT::f32)
11269 return Register();
11270 if ((Subtarget->hasFPARMv8())) {
11271 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11272 }
11273 return Register();
11274}
11275
11276Register fastEmit_ISD_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11277 if (RetVT.SimpleTy != MVT::f64)
11278 return Register();
11279 if ((Subtarget->hasFPARMv8())) {
11280 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11281 }
11282 return Register();
11283}
11284
11285Register fastEmit_ISD_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11286 if (RetVT.SimpleTy != MVT::v4f16)
11287 return Register();
11288 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11289 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11290 }
11291 return Register();
11292}
11293
11294Register fastEmit_ISD_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11295 if (RetVT.SimpleTy != MVT::v8f16)
11296 return Register();
11297 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11298 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11299 }
11300 return Register();
11301}
11302
11303Register fastEmit_ISD_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11304 if (RetVT.SimpleTy != MVT::v2f32)
11305 return Register();
11306 if ((Subtarget->isNeonAvailable())) {
11307 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11308 }
11309 return Register();
11310}
11311
11312Register fastEmit_ISD_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11313 if (RetVT.SimpleTy != MVT::v4f32)
11314 return Register();
11315 if ((Subtarget->isNeonAvailable())) {
11316 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11317 }
11318 return Register();
11319}
11320
11321Register fastEmit_ISD_FMINNUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11322 if (RetVT.SimpleTy != MVT::v1f64)
11323 return Register();
11324 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11325}
11326
11327Register fastEmit_ISD_FMINNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11328 if (RetVT.SimpleTy != MVT::v2f64)
11329 return Register();
11330 if ((Subtarget->isNeonAvailable())) {
11331 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11332 }
11333 return Register();
11334}
11335
11336Register fastEmit_ISD_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11337 switch (VT.SimpleTy) {
11338 case MVT::f16: return fastEmit_ISD_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
11339 case MVT::f32: return fastEmit_ISD_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
11340 case MVT::f64: return fastEmit_ISD_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
11341 case MVT::v4f16: return fastEmit_ISD_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11342 case MVT::v8f16: return fastEmit_ISD_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11343 case MVT::v2f32: return fastEmit_ISD_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11344 case MVT::v4f32: return fastEmit_ISD_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11345 case MVT::v1f64: return fastEmit_ISD_FMINNUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11346 case MVT::v2f64: return fastEmit_ISD_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11347 default: return Register();
11348 }
11349}
11350
11351// FastEmit functions for ISD::FMINNUM_IEEE.
11352
11353Register fastEmit_ISD_FMINNUM_IEEE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11354 if (RetVT.SimpleTy != MVT::f16)
11355 return Register();
11356 if ((Subtarget->hasFullFP16())) {
11357 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11358 }
11359 return Register();
11360}
11361
11362Register fastEmit_ISD_FMINNUM_IEEE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11363 if (RetVT.SimpleTy != MVT::f32)
11364 return Register();
11365 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11366}
11367
11368Register fastEmit_ISD_FMINNUM_IEEE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11369 if (RetVT.SimpleTy != MVT::f64)
11370 return Register();
11371 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11372}
11373
11374Register fastEmit_ISD_FMINNUM_IEEE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11375 if (RetVT.SimpleTy != MVT::v4f16)
11376 return Register();
11377 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11378 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11379 }
11380 return Register();
11381}
11382
11383Register fastEmit_ISD_FMINNUM_IEEE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11384 if (RetVT.SimpleTy != MVT::v8f16)
11385 return Register();
11386 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11387 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11388 }
11389 return Register();
11390}
11391
11392Register fastEmit_ISD_FMINNUM_IEEE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11393 if (RetVT.SimpleTy != MVT::v2f32)
11394 return Register();
11395 if ((Subtarget->isNeonAvailable())) {
11396 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11397 }
11398 return Register();
11399}
11400
11401Register fastEmit_ISD_FMINNUM_IEEE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11402 if (RetVT.SimpleTy != MVT::v4f32)
11403 return Register();
11404 if ((Subtarget->isNeonAvailable())) {
11405 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11406 }
11407 return Register();
11408}
11409
11410Register fastEmit_ISD_FMINNUM_IEEE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11411 if (RetVT.SimpleTy != MVT::v2f64)
11412 return Register();
11413 if ((Subtarget->isNeonAvailable())) {
11414 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11415 }
11416 return Register();
11417}
11418
11419Register fastEmit_ISD_FMINNUM_IEEE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11420 switch (VT.SimpleTy) {
11421 case MVT::f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_f16_rr(RetVT, Op0, Op1);
11422 case MVT::f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_f32_rr(RetVT, Op0, Op1);
11423 case MVT::f64: return fastEmit_ISD_FMINNUM_IEEE_MVT_f64_rr(RetVT, Op0, Op1);
11424 case MVT::v4f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_v4f16_rr(RetVT, Op0, Op1);
11425 case MVT::v8f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_v8f16_rr(RetVT, Op0, Op1);
11426 case MVT::v2f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_v2f32_rr(RetVT, Op0, Op1);
11427 case MVT::v4f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_v4f32_rr(RetVT, Op0, Op1);
11428 case MVT::v2f64: return fastEmit_ISD_FMINNUM_IEEE_MVT_v2f64_rr(RetVT, Op0, Op1);
11429 default: return Register();
11430 }
11431}
11432
11433// FastEmit functions for ISD::FMUL.
11434
11435Register fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11436 if (RetVT.SimpleTy != MVT::f16)
11437 return Register();
11438 if ((Subtarget->hasFullFP16())) {
11439 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11440 }
11441 return Register();
11442}
11443
11444Register fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11445 if (RetVT.SimpleTy != MVT::f32)
11446 return Register();
11447 if ((Subtarget->hasFPARMv8())) {
11448 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11449 }
11450 return Register();
11451}
11452
11453Register fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11454 if (RetVT.SimpleTy != MVT::f64)
11455 return Register();
11456 if ((Subtarget->hasFPARMv8())) {
11457 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11458 }
11459 return Register();
11460}
11461
11462Register fastEmit_ISD_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11463 if (RetVT.SimpleTy != MVT::v4f16)
11464 return Register();
11465 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11466 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11467 }
11468 return Register();
11469}
11470
11471Register fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11472 if (RetVT.SimpleTy != MVT::v8f16)
11473 return Register();
11474 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11475 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11476 }
11477 return Register();
11478}
11479
11480Register fastEmit_ISD_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11481 if (RetVT.SimpleTy != MVT::v2f32)
11482 return Register();
11483 if ((Subtarget->isNeonAvailable())) {
11484 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11485 }
11486 return Register();
11487}
11488
11489Register fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11490 if (RetVT.SimpleTy != MVT::v4f32)
11491 return Register();
11492 if ((Subtarget->isNeonAvailable())) {
11493 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11494 }
11495 return Register();
11496}
11497
11498Register fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11499 if (RetVT.SimpleTy != MVT::v2f64)
11500 return Register();
11501 if ((Subtarget->isNeonAvailable())) {
11502 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11503 }
11504 return Register();
11505}
11506
11507Register fastEmit_ISD_FMUL_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11508 if (RetVT.SimpleTy != MVT::nxv8f16)
11509 return Register();
11510 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11511 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11512 }
11513 return Register();
11514}
11515
11516Register fastEmit_ISD_FMUL_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11517 if (RetVT.SimpleTy != MVT::nxv8bf16)
11518 return Register();
11519 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
11520 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFMUL_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11521 }
11522 return Register();
11523}
11524
11525Register fastEmit_ISD_FMUL_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11526 if (RetVT.SimpleTy != MVT::nxv4f32)
11527 return Register();
11528 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11529 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11530 }
11531 return Register();
11532}
11533
11534Register fastEmit_ISD_FMUL_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11535 if (RetVT.SimpleTy != MVT::nxv2f64)
11536 return Register();
11537 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11538 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11539 }
11540 return Register();
11541}
11542
11543Register fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11544 switch (VT.SimpleTy) {
11545 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
11546 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
11547 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
11548 case MVT::v4f16: return fastEmit_ISD_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
11549 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
11550 case MVT::v2f32: return fastEmit_ISD_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
11551 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
11552 case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
11553 case MVT::nxv8f16: return fastEmit_ISD_FMUL_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11554 case MVT::nxv8bf16: return fastEmit_ISD_FMUL_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11555 case MVT::nxv4f32: return fastEmit_ISD_FMUL_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11556 case MVT::nxv2f64: return fastEmit_ISD_FMUL_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11557 default: return Register();
11558 }
11559}
11560
11561// FastEmit functions for ISD::FSUB.
11562
11563Register fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11564 if (RetVT.SimpleTy != MVT::f16)
11565 return Register();
11566 if ((Subtarget->hasFullFP16())) {
11567 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11568 }
11569 return Register();
11570}
11571
11572Register fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11573 if (RetVT.SimpleTy != MVT::f32)
11574 return Register();
11575 if ((Subtarget->hasFPARMv8())) {
11576 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11577 }
11578 return Register();
11579}
11580
11581Register fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11582 if (RetVT.SimpleTy != MVT::f64)
11583 return Register();
11584 if ((Subtarget->hasFPARMv8())) {
11585 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11586 }
11587 return Register();
11588}
11589
11590Register fastEmit_ISD_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11591 if (RetVT.SimpleTy != MVT::v4f16)
11592 return Register();
11593 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11594 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11595 }
11596 return Register();
11597}
11598
11599Register fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11600 if (RetVT.SimpleTy != MVT::v8f16)
11601 return Register();
11602 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11603 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11604 }
11605 return Register();
11606}
11607
11608Register fastEmit_ISD_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11609 if (RetVT.SimpleTy != MVT::v2f32)
11610 return Register();
11611 if ((Subtarget->isNeonAvailable())) {
11612 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11613 }
11614 return Register();
11615}
11616
11617Register fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11618 if (RetVT.SimpleTy != MVT::v4f32)
11619 return Register();
11620 if ((Subtarget->isNeonAvailable())) {
11621 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11622 }
11623 return Register();
11624}
11625
11626Register fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11627 if (RetVT.SimpleTy != MVT::v2f64)
11628 return Register();
11629 if ((Subtarget->isNeonAvailable())) {
11630 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11631 }
11632 return Register();
11633}
11634
11635Register fastEmit_ISD_FSUB_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11636 if (RetVT.SimpleTy != MVT::nxv8f16)
11637 return Register();
11638 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11639 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11640 }
11641 return Register();
11642}
11643
11644Register fastEmit_ISD_FSUB_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11645 if (RetVT.SimpleTy != MVT::nxv8bf16)
11646 return Register();
11647 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
11648 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFSUB_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11649 }
11650 return Register();
11651}
11652
11653Register fastEmit_ISD_FSUB_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11654 if (RetVT.SimpleTy != MVT::nxv4f32)
11655 return Register();
11656 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11657 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11658 }
11659 return Register();
11660}
11661
11662Register fastEmit_ISD_FSUB_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11663 if (RetVT.SimpleTy != MVT::nxv2f64)
11664 return Register();
11665 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11666 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11667 }
11668 return Register();
11669}
11670
11671Register fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11672 switch (VT.SimpleTy) {
11673 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
11674 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
11675 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
11676 case MVT::v4f16: return fastEmit_ISD_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
11677 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
11678 case MVT::v2f32: return fastEmit_ISD_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
11679 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
11680 case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
11681 case MVT::nxv8f16: return fastEmit_ISD_FSUB_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11682 case MVT::nxv8bf16: return fastEmit_ISD_FSUB_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11683 case MVT::nxv4f32: return fastEmit_ISD_FSUB_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11684 case MVT::nxv2f64: return fastEmit_ISD_FSUB_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11685 default: return Register();
11686 }
11687}
11688
11689// FastEmit functions for ISD::GET_ACTIVE_LANE_MASK.
11690
11691Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv2i1_rr(Register Op0, Register Op1) {
11692 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11693 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_D, RC: &AArch64::PPRRegClass, Op0, Op1);
11694 }
11695 return Register();
11696}
11697
11698Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv4i1_rr(Register Op0, Register Op1) {
11699 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11700 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_S, RC: &AArch64::PPRRegClass, Op0, Op1);
11701 }
11702 return Register();
11703}
11704
11705Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv8i1_rr(Register Op0, Register Op1) {
11706 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11707 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_H, RC: &AArch64::PPRRegClass, Op0, Op1);
11708 }
11709 return Register();
11710}
11711
11712Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv16i1_rr(Register Op0, Register Op1) {
11713 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11714 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_B, RC: &AArch64::PPRRegClass, Op0, Op1);
11715 }
11716 return Register();
11717}
11718
11719Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
11720switch (RetVT.SimpleTy) {
11721 case MVT::nxv2i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv2i1_rr(Op0, Op1);
11722 case MVT::nxv4i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv4i1_rr(Op0, Op1);
11723 case MVT::nxv8i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv8i1_rr(Op0, Op1);
11724 case MVT::nxv16i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv16i1_rr(Op0, Op1);
11725 default: return Register();
11726}
11727}
11728
11729Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv2i1_rr(Register Op0, Register Op1) {
11730 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11731 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_D, RC: &AArch64::PPRRegClass, Op0, Op1);
11732 }
11733 return Register();
11734}
11735
11736Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv4i1_rr(Register Op0, Register Op1) {
11737 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11738 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_S, RC: &AArch64::PPRRegClass, Op0, Op1);
11739 }
11740 return Register();
11741}
11742
11743Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv8i1_rr(Register Op0, Register Op1) {
11744 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11745 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_H, RC: &AArch64::PPRRegClass, Op0, Op1);
11746 }
11747 return Register();
11748}
11749
11750Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv16i1_rr(Register Op0, Register Op1) {
11751 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11752 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_B, RC: &AArch64::PPRRegClass, Op0, Op1);
11753 }
11754 return Register();
11755}
11756
11757Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11758switch (RetVT.SimpleTy) {
11759 case MVT::nxv2i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv2i1_rr(Op0, Op1);
11760 case MVT::nxv4i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv4i1_rr(Op0, Op1);
11761 case MVT::nxv8i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv8i1_rr(Op0, Op1);
11762 case MVT::nxv16i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv16i1_rr(Op0, Op1);
11763 default: return Register();
11764}
11765}
11766
11767Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11768 switch (VT.SimpleTy) {
11769 case MVT::i32: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_rr(RetVT, Op0, Op1);
11770 case MVT::i64: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_rr(RetVT, Op0, Op1);
11771 default: return Register();
11772 }
11773}
11774
11775// FastEmit functions for ISD::MUL.
11776
11777Register fastEmit_ISD_MUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
11778 if (RetVT.SimpleTy != MVT::v8i8)
11779 return Register();
11780 if ((Subtarget->isNeonAvailable())) {
11781 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
11782 }
11783 return Register();
11784}
11785
11786Register fastEmit_ISD_MUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
11787 if (RetVT.SimpleTy != MVT::v16i8)
11788 return Register();
11789 if ((Subtarget->isNeonAvailable())) {
11790 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
11791 }
11792 return Register();
11793}
11794
11795Register fastEmit_ISD_MUL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
11796 if (RetVT.SimpleTy != MVT::v4i16)
11797 return Register();
11798 if ((Subtarget->isNeonAvailable())) {
11799 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11800 }
11801 return Register();
11802}
11803
11804Register fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
11805 if (RetVT.SimpleTy != MVT::v8i16)
11806 return Register();
11807 if ((Subtarget->isNeonAvailable())) {
11808 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11809 }
11810 return Register();
11811}
11812
11813Register fastEmit_ISD_MUL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
11814 if (RetVT.SimpleTy != MVT::v2i32)
11815 return Register();
11816 if ((Subtarget->isNeonAvailable())) {
11817 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11818 }
11819 return Register();
11820}
11821
11822Register fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
11823 if (RetVT.SimpleTy != MVT::v4i32)
11824 return Register();
11825 if ((Subtarget->isNeonAvailable())) {
11826 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11827 }
11828 return Register();
11829}
11830
11831Register fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11832 switch (VT.SimpleTy) {
11833 case MVT::v8i8: return fastEmit_ISD_MUL_MVT_v8i8_rr(RetVT, Op0, Op1);
11834 case MVT::v16i8: return fastEmit_ISD_MUL_MVT_v16i8_rr(RetVT, Op0, Op1);
11835 case MVT::v4i16: return fastEmit_ISD_MUL_MVT_v4i16_rr(RetVT, Op0, Op1);
11836 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
11837 case MVT::v2i32: return fastEmit_ISD_MUL_MVT_v2i32_rr(RetVT, Op0, Op1);
11838 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
11839 default: return Register();
11840 }
11841}
11842
11843// FastEmit functions for ISD::MULHS.
11844
11845Register fastEmit_ISD_MULHS_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11846 if (RetVT.SimpleTy != MVT::i64)
11847 return Register();
11848 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULHrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
11849}
11850
11851Register fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11852 switch (VT.SimpleTy) {
11853 case MVT::i64: return fastEmit_ISD_MULHS_MVT_i64_rr(RetVT, Op0, Op1);
11854 default: return Register();
11855 }
11856}
11857
11858// FastEmit functions for ISD::MULHU.
11859
11860Register fastEmit_ISD_MULHU_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11861 if (RetVT.SimpleTy != MVT::i64)
11862 return Register();
11863 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULHrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
11864}
11865
11866Register fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11867 switch (VT.SimpleTy) {
11868 case MVT::i64: return fastEmit_ISD_MULHU_MVT_i64_rr(RetVT, Op0, Op1);
11869 default: return Register();
11870 }
11871}
11872
11873// FastEmit functions for ISD::OR.
11874
11875Register fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
11876 if (RetVT.SimpleTy != MVT::i32)
11877 return Register();
11878 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
11879}
11880
11881Register fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11882 if (RetVT.SimpleTy != MVT::i64)
11883 return Register();
11884 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
11885}
11886
11887Register fastEmit_ISD_OR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
11888 if (RetVT.SimpleTy != MVT::v8i8)
11889 return Register();
11890 if ((Subtarget->isNeonAvailable())) {
11891 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
11892 }
11893 return Register();
11894}
11895
11896Register fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
11897 if (RetVT.SimpleTy != MVT::v16i8)
11898 return Register();
11899 if ((Subtarget->isNeonAvailable())) {
11900 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
11901 }
11902 return Register();
11903}
11904
11905Register fastEmit_ISD_OR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
11906 if (RetVT.SimpleTy != MVT::v4i16)
11907 return Register();
11908 if ((Subtarget->isNeonAvailable())) {
11909 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
11910 }
11911 return Register();
11912}
11913
11914Register fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
11915 if (RetVT.SimpleTy != MVT::v8i16)
11916 return Register();
11917 if ((Subtarget->isNeonAvailable())) {
11918 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
11919 }
11920 return Register();
11921}
11922
11923Register fastEmit_ISD_OR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
11924 if (RetVT.SimpleTy != MVT::v2i32)
11925 return Register();
11926 if ((Subtarget->isNeonAvailable())) {
11927 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
11928 }
11929 return Register();
11930}
11931
11932Register fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
11933 if (RetVT.SimpleTy != MVT::v4i32)
11934 return Register();
11935 if ((Subtarget->isNeonAvailable())) {
11936 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
11937 }
11938 return Register();
11939}
11940
11941Register fastEmit_ISD_OR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
11942 if (RetVT.SimpleTy != MVT::v1i64)
11943 return Register();
11944 if ((Subtarget->isNeonAvailable())) {
11945 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
11946 }
11947 return Register();
11948}
11949
11950Register fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
11951 if (RetVT.SimpleTy != MVT::v2i64)
11952 return Register();
11953 if ((Subtarget->isNeonAvailable())) {
11954 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
11955 }
11956 return Register();
11957}
11958
11959Register fastEmit_ISD_OR_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
11960 if (RetVT.SimpleTy != MVT::nxv16i8)
11961 return Register();
11962 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11963 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11964 }
11965 return Register();
11966}
11967
11968Register fastEmit_ISD_OR_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
11969 if (RetVT.SimpleTy != MVT::nxv8i16)
11970 return Register();
11971 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11972 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11973 }
11974 return Register();
11975}
11976
11977Register fastEmit_ISD_OR_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
11978 if (RetVT.SimpleTy != MVT::nxv4i32)
11979 return Register();
11980 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11981 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11982 }
11983 return Register();
11984}
11985
11986Register fastEmit_ISD_OR_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
11987 if (RetVT.SimpleTy != MVT::nxv2i64)
11988 return Register();
11989 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11990 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11991 }
11992 return Register();
11993}
11994
11995Register fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11996 switch (VT.SimpleTy) {
11997 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
11998 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op1);
11999 case MVT::v8i8: return fastEmit_ISD_OR_MVT_v8i8_rr(RetVT, Op0, Op1);
12000 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
12001 case MVT::v4i16: return fastEmit_ISD_OR_MVT_v4i16_rr(RetVT, Op0, Op1);
12002 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
12003 case MVT::v2i32: return fastEmit_ISD_OR_MVT_v2i32_rr(RetVT, Op0, Op1);
12004 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
12005 case MVT::v1i64: return fastEmit_ISD_OR_MVT_v1i64_rr(RetVT, Op0, Op1);
12006 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
12007 case MVT::nxv16i8: return fastEmit_ISD_OR_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12008 case MVT::nxv8i16: return fastEmit_ISD_OR_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12009 case MVT::nxv4i32: return fastEmit_ISD_OR_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12010 case MVT::nxv2i64: return fastEmit_ISD_OR_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12011 default: return Register();
12012 }
12013}
12014
12015// FastEmit functions for ISD::ROTR.
12016
12017Register fastEmit_ISD_ROTR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12018 if (RetVT.SimpleTy != MVT::i64)
12019 return Register();
12020 return fastEmitInst_rr(MachineInstOpcode: AArch64::RORVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12021}
12022
12023Register fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12024 switch (VT.SimpleTy) {
12025 case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_rr(RetVT, Op0, Op1);
12026 default: return Register();
12027 }
12028}
12029
12030// FastEmit functions for ISD::SADDSAT.
12031
12032Register fastEmit_ISD_SADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12033 if (RetVT.SimpleTy != MVT::v8i8)
12034 return Register();
12035 if ((Subtarget->isNeonAvailable())) {
12036 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12037 }
12038 return Register();
12039}
12040
12041Register fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12042 if (RetVT.SimpleTy != MVT::v16i8)
12043 return Register();
12044 if ((Subtarget->isNeonAvailable())) {
12045 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12046 }
12047 return Register();
12048}
12049
12050Register fastEmit_ISD_SADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12051 if (RetVT.SimpleTy != MVT::v4i16)
12052 return Register();
12053 if ((Subtarget->isNeonAvailable())) {
12054 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12055 }
12056 return Register();
12057}
12058
12059Register fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12060 if (RetVT.SimpleTy != MVT::v8i16)
12061 return Register();
12062 if ((Subtarget->isNeonAvailable())) {
12063 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12064 }
12065 return Register();
12066}
12067
12068Register fastEmit_ISD_SADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12069 if (RetVT.SimpleTy != MVT::v2i32)
12070 return Register();
12071 if ((Subtarget->isNeonAvailable())) {
12072 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12073 }
12074 return Register();
12075}
12076
12077Register fastEmit_ISD_SADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12078 if (RetVT.SimpleTy != MVT::v4i32)
12079 return Register();
12080 if ((Subtarget->isNeonAvailable())) {
12081 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12082 }
12083 return Register();
12084}
12085
12086Register fastEmit_ISD_SADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12087 if (RetVT.SimpleTy != MVT::v1i64)
12088 return Register();
12089 if ((Subtarget->isNeonAvailable())) {
12090 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
12091 }
12092 return Register();
12093}
12094
12095Register fastEmit_ISD_SADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12096 if (RetVT.SimpleTy != MVT::v2i64)
12097 return Register();
12098 if ((Subtarget->isNeonAvailable())) {
12099 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12100 }
12101 return Register();
12102}
12103
12104Register fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12105 if (RetVT.SimpleTy != MVT::nxv16i8)
12106 return Register();
12107 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12108 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
12109 }
12110 return Register();
12111}
12112
12113Register fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12114 if (RetVT.SimpleTy != MVT::nxv8i16)
12115 return Register();
12116 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12117 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
12118 }
12119 return Register();
12120}
12121
12122Register fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12123 if (RetVT.SimpleTy != MVT::nxv4i32)
12124 return Register();
12125 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12126 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
12127 }
12128 return Register();
12129}
12130
12131Register fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12132 if (RetVT.SimpleTy != MVT::nxv2i64)
12133 return Register();
12134 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12135 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
12136 }
12137 return Register();
12138}
12139
12140Register fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12141 switch (VT.SimpleTy) {
12142 case MVT::v8i8: return fastEmit_ISD_SADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
12143 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12144 case MVT::v4i16: return fastEmit_ISD_SADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
12145 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12146 case MVT::v2i32: return fastEmit_ISD_SADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
12147 case MVT::v4i32: return fastEmit_ISD_SADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
12148 case MVT::v1i64: return fastEmit_ISD_SADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
12149 case MVT::v2i64: return fastEmit_ISD_SADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
12150 case MVT::nxv16i8: return fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12151 case MVT::nxv8i16: return fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12152 case MVT::nxv4i32: return fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12153 case MVT::nxv2i64: return fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12154 default: return Register();
12155 }
12156}
12157
12158// FastEmit functions for ISD::SDIV.
12159
12160Register fastEmit_ISD_SDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12161 if (RetVT.SimpleTy != MVT::i32)
12162 return Register();
12163 return fastEmitInst_rr(MachineInstOpcode: AArch64::SDIVWr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12164}
12165
12166Register fastEmit_ISD_SDIV_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12167 if (RetVT.SimpleTy != MVT::i64)
12168 return Register();
12169 return fastEmitInst_rr(MachineInstOpcode: AArch64::SDIVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12170}
12171
12172Register fastEmit_ISD_SDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12173 switch (VT.SimpleTy) {
12174 case MVT::i32: return fastEmit_ISD_SDIV_MVT_i32_rr(RetVT, Op0, Op1);
12175 case MVT::i64: return fastEmit_ISD_SDIV_MVT_i64_rr(RetVT, Op0, Op1);
12176 default: return Register();
12177 }
12178}
12179
12180// FastEmit functions for ISD::SHL.
12181
12182Register fastEmit_ISD_SHL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12183 if (RetVT.SimpleTy != MVT::i64)
12184 return Register();
12185 return fastEmitInst_rr(MachineInstOpcode: AArch64::LSLVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12186}
12187
12188Register fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12189 switch (VT.SimpleTy) {
12190 case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_rr(RetVT, Op0, Op1);
12191 default: return Register();
12192 }
12193}
12194
12195// FastEmit functions for ISD::SMAX.
12196
12197Register fastEmit_ISD_SMAX_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12198 if (RetVT.SimpleTy != MVT::i32)
12199 return Register();
12200 if ((Subtarget->hasCSSC())) {
12201 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12202 }
12203 return Register();
12204}
12205
12206Register fastEmit_ISD_SMAX_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12207 if (RetVT.SimpleTy != MVT::i64)
12208 return Register();
12209 if ((Subtarget->hasCSSC())) {
12210 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12211 }
12212 return Register();
12213}
12214
12215Register fastEmit_ISD_SMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12216 if (RetVT.SimpleTy != MVT::v8i8)
12217 return Register();
12218 if ((Subtarget->isNeonAvailable())) {
12219 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12220 }
12221 return Register();
12222}
12223
12224Register fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12225 if (RetVT.SimpleTy != MVT::v16i8)
12226 return Register();
12227 if ((Subtarget->isNeonAvailable())) {
12228 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12229 }
12230 return Register();
12231}
12232
12233Register fastEmit_ISD_SMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12234 if (RetVT.SimpleTy != MVT::v4i16)
12235 return Register();
12236 if ((Subtarget->isNeonAvailable())) {
12237 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12238 }
12239 return Register();
12240}
12241
12242Register fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12243 if (RetVT.SimpleTy != MVT::v8i16)
12244 return Register();
12245 if ((Subtarget->isNeonAvailable())) {
12246 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12247 }
12248 return Register();
12249}
12250
12251Register fastEmit_ISD_SMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12252 if (RetVT.SimpleTy != MVT::v2i32)
12253 return Register();
12254 if ((Subtarget->isNeonAvailable())) {
12255 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12256 }
12257 return Register();
12258}
12259
12260Register fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12261 if (RetVT.SimpleTy != MVT::v4i32)
12262 return Register();
12263 if ((Subtarget->isNeonAvailable())) {
12264 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12265 }
12266 return Register();
12267}
12268
12269Register fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12270 switch (VT.SimpleTy) {
12271 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_rr(RetVT, Op0, Op1);
12272 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_rr(RetVT, Op0, Op1);
12273 case MVT::v8i8: return fastEmit_ISD_SMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
12274 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
12275 case MVT::v4i16: return fastEmit_ISD_SMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
12276 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
12277 case MVT::v2i32: return fastEmit_ISD_SMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
12278 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
12279 default: return Register();
12280 }
12281}
12282
12283// FastEmit functions for ISD::SMIN.
12284
12285Register fastEmit_ISD_SMIN_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12286 if (RetVT.SimpleTy != MVT::i32)
12287 return Register();
12288 if ((Subtarget->hasCSSC())) {
12289 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12290 }
12291 return Register();
12292}
12293
12294Register fastEmit_ISD_SMIN_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12295 if (RetVT.SimpleTy != MVT::i64)
12296 return Register();
12297 if ((Subtarget->hasCSSC())) {
12298 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12299 }
12300 return Register();
12301}
12302
12303Register fastEmit_ISD_SMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12304 if (RetVT.SimpleTy != MVT::v8i8)
12305 return Register();
12306 if ((Subtarget->isNeonAvailable())) {
12307 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12308 }
12309 return Register();
12310}
12311
12312Register fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12313 if (RetVT.SimpleTy != MVT::v16i8)
12314 return Register();
12315 if ((Subtarget->isNeonAvailable())) {
12316 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12317 }
12318 return Register();
12319}
12320
12321Register fastEmit_ISD_SMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12322 if (RetVT.SimpleTy != MVT::v4i16)
12323 return Register();
12324 if ((Subtarget->isNeonAvailable())) {
12325 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12326 }
12327 return Register();
12328}
12329
12330Register fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12331 if (RetVT.SimpleTy != MVT::v8i16)
12332 return Register();
12333 if ((Subtarget->isNeonAvailable())) {
12334 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12335 }
12336 return Register();
12337}
12338
12339Register fastEmit_ISD_SMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12340 if (RetVT.SimpleTy != MVT::v2i32)
12341 return Register();
12342 if ((Subtarget->isNeonAvailable())) {
12343 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12344 }
12345 return Register();
12346}
12347
12348Register fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12349 if (RetVT.SimpleTy != MVT::v4i32)
12350 return Register();
12351 if ((Subtarget->isNeonAvailable())) {
12352 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12353 }
12354 return Register();
12355}
12356
12357Register fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12358 switch (VT.SimpleTy) {
12359 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_rr(RetVT, Op0, Op1);
12360 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_rr(RetVT, Op0, Op1);
12361 case MVT::v8i8: return fastEmit_ISD_SMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
12362 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
12363 case MVT::v4i16: return fastEmit_ISD_SMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
12364 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
12365 case MVT::v2i32: return fastEmit_ISD_SMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
12366 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
12367 default: return Register();
12368 }
12369}
12370
12371// FastEmit functions for ISD::SRA.
12372
12373Register fastEmit_ISD_SRA_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12374 if (RetVT.SimpleTy != MVT::i64)
12375 return Register();
12376 return fastEmitInst_rr(MachineInstOpcode: AArch64::ASRVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12377}
12378
12379Register fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12380 switch (VT.SimpleTy) {
12381 case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_rr(RetVT, Op0, Op1);
12382 default: return Register();
12383 }
12384}
12385
12386// FastEmit functions for ISD::SRL.
12387
12388Register fastEmit_ISD_SRL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12389 if (RetVT.SimpleTy != MVT::i64)
12390 return Register();
12391 return fastEmitInst_rr(MachineInstOpcode: AArch64::LSRVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12392}
12393
12394Register fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12395 switch (VT.SimpleTy) {
12396 case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_rr(RetVT, Op0, Op1);
12397 default: return Register();
12398 }
12399}
12400
12401// FastEmit functions for ISD::SSUBSAT.
12402
12403Register fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12404 if (RetVT.SimpleTy != MVT::v8i8)
12405 return Register();
12406 if ((Subtarget->isNeonAvailable())) {
12407 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12408 }
12409 return Register();
12410}
12411
12412Register fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12413 if (RetVT.SimpleTy != MVT::v16i8)
12414 return Register();
12415 if ((Subtarget->isNeonAvailable())) {
12416 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12417 }
12418 return Register();
12419}
12420
12421Register fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12422 if (RetVT.SimpleTy != MVT::v4i16)
12423 return Register();
12424 if ((Subtarget->isNeonAvailable())) {
12425 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12426 }
12427 return Register();
12428}
12429
12430Register fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12431 if (RetVT.SimpleTy != MVT::v8i16)
12432 return Register();
12433 if ((Subtarget->isNeonAvailable())) {
12434 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12435 }
12436 return Register();
12437}
12438
12439Register fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12440 if (RetVT.SimpleTy != MVT::v2i32)
12441 return Register();
12442 if ((Subtarget->isNeonAvailable())) {
12443 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12444 }
12445 return Register();
12446}
12447
12448Register fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12449 if (RetVT.SimpleTy != MVT::v4i32)
12450 return Register();
12451 if ((Subtarget->isNeonAvailable())) {
12452 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12453 }
12454 return Register();
12455}
12456
12457Register fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12458 if (RetVT.SimpleTy != MVT::v1i64)
12459 return Register();
12460 if ((Subtarget->isNeonAvailable())) {
12461 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
12462 }
12463 return Register();
12464}
12465
12466Register fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12467 if (RetVT.SimpleTy != MVT::v2i64)
12468 return Register();
12469 if ((Subtarget->isNeonAvailable())) {
12470 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12471 }
12472 return Register();
12473}
12474
12475Register fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12476 if (RetVT.SimpleTy != MVT::nxv16i8)
12477 return Register();
12478 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12479 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
12480 }
12481 return Register();
12482}
12483
12484Register fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12485 if (RetVT.SimpleTy != MVT::nxv8i16)
12486 return Register();
12487 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12488 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
12489 }
12490 return Register();
12491}
12492
12493Register fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12494 if (RetVT.SimpleTy != MVT::nxv4i32)
12495 return Register();
12496 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12497 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
12498 }
12499 return Register();
12500}
12501
12502Register fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12503 if (RetVT.SimpleTy != MVT::nxv2i64)
12504 return Register();
12505 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12506 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
12507 }
12508 return Register();
12509}
12510
12511Register fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12512 switch (VT.SimpleTy) {
12513 case MVT::v8i8: return fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
12514 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12515 case MVT::v4i16: return fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
12516 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12517 case MVT::v2i32: return fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
12518 case MVT::v4i32: return fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
12519 case MVT::v1i64: return fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
12520 case MVT::v2i64: return fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
12521 case MVT::nxv16i8: return fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12522 case MVT::nxv8i16: return fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12523 case MVT::nxv4i32: return fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12524 case MVT::nxv2i64: return fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12525 default: return Register();
12526 }
12527}
12528
12529// FastEmit functions for ISD::STRICT_FADD.
12530
12531Register fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12532 if (RetVT.SimpleTy != MVT::f16)
12533 return Register();
12534 if ((Subtarget->hasFullFP16())) {
12535 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12536 }
12537 return Register();
12538}
12539
12540Register fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12541 if (RetVT.SimpleTy != MVT::f32)
12542 return Register();
12543 if ((Subtarget->hasFPARMv8())) {
12544 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12545 }
12546 return Register();
12547}
12548
12549Register fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12550 if (RetVT.SimpleTy != MVT::f64)
12551 return Register();
12552 if ((Subtarget->hasFPARMv8())) {
12553 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12554 }
12555 return Register();
12556}
12557
12558Register fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12559 if (RetVT.SimpleTy != MVT::v4f16)
12560 return Register();
12561 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12562 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12563 }
12564 return Register();
12565}
12566
12567Register fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12568 if (RetVT.SimpleTy != MVT::v8f16)
12569 return Register();
12570 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12571 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12572 }
12573 return Register();
12574}
12575
12576Register fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12577 if (RetVT.SimpleTy != MVT::v2f32)
12578 return Register();
12579 if ((Subtarget->isNeonAvailable())) {
12580 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12581 }
12582 return Register();
12583}
12584
12585Register fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12586 if (RetVT.SimpleTy != MVT::v4f32)
12587 return Register();
12588 if ((Subtarget->isNeonAvailable())) {
12589 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12590 }
12591 return Register();
12592}
12593
12594Register fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12595 if (RetVT.SimpleTy != MVT::v2f64)
12596 return Register();
12597 if ((Subtarget->isNeonAvailable())) {
12598 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12599 }
12600 return Register();
12601}
12602
12603Register fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12604 switch (VT.SimpleTy) {
12605 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1);
12606 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1);
12607 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1);
12608 case MVT::v4f16: return fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
12609 case MVT::v8f16: return fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
12610 case MVT::v2f32: return fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
12611 case MVT::v4f32: return fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
12612 case MVT::v2f64: return fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
12613 default: return Register();
12614 }
12615}
12616
12617// FastEmit functions for ISD::STRICT_FDIV.
12618
12619Register fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12620 if (RetVT.SimpleTy != MVT::f16)
12621 return Register();
12622 if ((Subtarget->hasFullFP16())) {
12623 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12624 }
12625 return Register();
12626}
12627
12628Register fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12629 if (RetVT.SimpleTy != MVT::f32)
12630 return Register();
12631 if ((Subtarget->hasFPARMv8())) {
12632 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12633 }
12634 return Register();
12635}
12636
12637Register fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12638 if (RetVT.SimpleTy != MVT::f64)
12639 return Register();
12640 if ((Subtarget->hasFPARMv8())) {
12641 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12642 }
12643 return Register();
12644}
12645
12646Register fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12647 if (RetVT.SimpleTy != MVT::v4f16)
12648 return Register();
12649 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12650 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12651 }
12652 return Register();
12653}
12654
12655Register fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12656 if (RetVT.SimpleTy != MVT::v8f16)
12657 return Register();
12658 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12659 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12660 }
12661 return Register();
12662}
12663
12664Register fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12665 if (RetVT.SimpleTy != MVT::v2f32)
12666 return Register();
12667 if ((Subtarget->isNeonAvailable())) {
12668 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12669 }
12670 return Register();
12671}
12672
12673Register fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12674 if (RetVT.SimpleTy != MVT::v4f32)
12675 return Register();
12676 if ((Subtarget->isNeonAvailable())) {
12677 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12678 }
12679 return Register();
12680}
12681
12682Register fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12683 if (RetVT.SimpleTy != MVT::v2f64)
12684 return Register();
12685 if ((Subtarget->isNeonAvailable())) {
12686 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12687 }
12688 return Register();
12689}
12690
12691Register fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12692 switch (VT.SimpleTy) {
12693 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
12694 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
12695 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
12696 case MVT::v4f16: return fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1);
12697 case MVT::v8f16: return fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
12698 case MVT::v2f32: return fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1);
12699 case MVT::v4f32: return fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
12700 case MVT::v2f64: return fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
12701 default: return Register();
12702 }
12703}
12704
12705// FastEmit functions for ISD::STRICT_FMAXIMUM.
12706
12707Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12708 if (RetVT.SimpleTy != MVT::f16)
12709 return Register();
12710 if ((Subtarget->hasFullFP16())) {
12711 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12712 }
12713 return Register();
12714}
12715
12716Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12717 if (RetVT.SimpleTy != MVT::f32)
12718 return Register();
12719 if ((Subtarget->hasFPARMv8())) {
12720 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12721 }
12722 return Register();
12723}
12724
12725Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12726 if (RetVT.SimpleTy != MVT::f64)
12727 return Register();
12728 if ((Subtarget->hasFPARMv8())) {
12729 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12730 }
12731 return Register();
12732}
12733
12734Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12735 if (RetVT.SimpleTy != MVT::v4f16)
12736 return Register();
12737 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12738 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12739 }
12740 return Register();
12741}
12742
12743Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12744 if (RetVT.SimpleTy != MVT::v8f16)
12745 return Register();
12746 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12747 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12748 }
12749 return Register();
12750}
12751
12752Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12753 if (RetVT.SimpleTy != MVT::v2f32)
12754 return Register();
12755 if ((Subtarget->isNeonAvailable())) {
12756 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12757 }
12758 return Register();
12759}
12760
12761Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12762 if (RetVT.SimpleTy != MVT::v4f32)
12763 return Register();
12764 if ((Subtarget->isNeonAvailable())) {
12765 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12766 }
12767 return Register();
12768}
12769
12770Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12771 if (RetVT.SimpleTy != MVT::v2f64)
12772 return Register();
12773 if ((Subtarget->isNeonAvailable())) {
12774 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12775 }
12776 return Register();
12777}
12778
12779Register fastEmit_ISD_STRICT_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12780 switch (VT.SimpleTy) {
12781 case MVT::f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1);
12782 case MVT::f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1);
12783 case MVT::f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1);
12784 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
12785 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
12786 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
12787 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
12788 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
12789 default: return Register();
12790 }
12791}
12792
12793// FastEmit functions for ISD::STRICT_FMAXNUM.
12794
12795Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12796 if (RetVT.SimpleTy != MVT::f16)
12797 return Register();
12798 if ((Subtarget->hasFullFP16())) {
12799 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12800 }
12801 return Register();
12802}
12803
12804Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12805 if (RetVT.SimpleTy != MVT::f32)
12806 return Register();
12807 if ((Subtarget->hasFPARMv8())) {
12808 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12809 }
12810 return Register();
12811}
12812
12813Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12814 if (RetVT.SimpleTy != MVT::f64)
12815 return Register();
12816 if ((Subtarget->hasFPARMv8())) {
12817 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12818 }
12819 return Register();
12820}
12821
12822Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12823 if (RetVT.SimpleTy != MVT::v4f16)
12824 return Register();
12825 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12826 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12827 }
12828 return Register();
12829}
12830
12831Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12832 if (RetVT.SimpleTy != MVT::v8f16)
12833 return Register();
12834 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12835 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12836 }
12837 return Register();
12838}
12839
12840Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12841 if (RetVT.SimpleTy != MVT::v2f32)
12842 return Register();
12843 if ((Subtarget->isNeonAvailable())) {
12844 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12845 }
12846 return Register();
12847}
12848
12849Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12850 if (RetVT.SimpleTy != MVT::v4f32)
12851 return Register();
12852 if ((Subtarget->isNeonAvailable())) {
12853 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12854 }
12855 return Register();
12856}
12857
12858Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12859 if (RetVT.SimpleTy != MVT::v2f64)
12860 return Register();
12861 if ((Subtarget->isNeonAvailable())) {
12862 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12863 }
12864 return Register();
12865}
12866
12867Register fastEmit_ISD_STRICT_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12868 switch (VT.SimpleTy) {
12869 case MVT::f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
12870 case MVT::f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
12871 case MVT::f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
12872 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
12873 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
12874 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
12875 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
12876 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
12877 default: return Register();
12878 }
12879}
12880
12881// FastEmit functions for ISD::STRICT_FMINIMUM.
12882
12883Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12884 if (RetVT.SimpleTy != MVT::f16)
12885 return Register();
12886 if ((Subtarget->hasFullFP16())) {
12887 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12888 }
12889 return Register();
12890}
12891
12892Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12893 if (RetVT.SimpleTy != MVT::f32)
12894 return Register();
12895 if ((Subtarget->hasFPARMv8())) {
12896 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12897 }
12898 return Register();
12899}
12900
12901Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12902 if (RetVT.SimpleTy != MVT::f64)
12903 return Register();
12904 if ((Subtarget->hasFPARMv8())) {
12905 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12906 }
12907 return Register();
12908}
12909
12910Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12911 if (RetVT.SimpleTy != MVT::v4f16)
12912 return Register();
12913 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12914 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12915 }
12916 return Register();
12917}
12918
12919Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12920 if (RetVT.SimpleTy != MVT::v8f16)
12921 return Register();
12922 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12923 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12924 }
12925 return Register();
12926}
12927
12928Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12929 if (RetVT.SimpleTy != MVT::v2f32)
12930 return Register();
12931 if ((Subtarget->isNeonAvailable())) {
12932 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12933 }
12934 return Register();
12935}
12936
12937Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12938 if (RetVT.SimpleTy != MVT::v4f32)
12939 return Register();
12940 if ((Subtarget->isNeonAvailable())) {
12941 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12942 }
12943 return Register();
12944}
12945
12946Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12947 if (RetVT.SimpleTy != MVT::v2f64)
12948 return Register();
12949 if ((Subtarget->isNeonAvailable())) {
12950 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12951 }
12952 return Register();
12953}
12954
12955Register fastEmit_ISD_STRICT_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12956 switch (VT.SimpleTy) {
12957 case MVT::f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1);
12958 case MVT::f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1);
12959 case MVT::f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1);
12960 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
12961 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
12962 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
12963 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
12964 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
12965 default: return Register();
12966 }
12967}
12968
12969// FastEmit functions for ISD::STRICT_FMINNUM.
12970
12971Register fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12972 if (RetVT.SimpleTy != MVT::f16)
12973 return Register();
12974 if ((Subtarget->hasFullFP16())) {
12975 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12976 }
12977 return Register();
12978}
12979
12980Register fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12981 if (RetVT.SimpleTy != MVT::f32)
12982 return Register();
12983 if ((Subtarget->hasFPARMv8())) {
12984 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12985 }
12986 return Register();
12987}
12988
12989Register fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12990 if (RetVT.SimpleTy != MVT::f64)
12991 return Register();
12992 if ((Subtarget->hasFPARMv8())) {
12993 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12994 }
12995 return Register();
12996}
12997
12998Register fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12999 if (RetVT.SimpleTy != MVT::v4f16)
13000 return Register();
13001 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13002 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13003 }
13004 return Register();
13005}
13006
13007Register fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13008 if (RetVT.SimpleTy != MVT::v8f16)
13009 return Register();
13010 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13011 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13012 }
13013 return Register();
13014}
13015
13016Register fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13017 if (RetVT.SimpleTy != MVT::v2f32)
13018 return Register();
13019 if ((Subtarget->isNeonAvailable())) {
13020 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13021 }
13022 return Register();
13023}
13024
13025Register fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13026 if (RetVT.SimpleTy != MVT::v4f32)
13027 return Register();
13028 if ((Subtarget->isNeonAvailable())) {
13029 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13030 }
13031 return Register();
13032}
13033
13034Register fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13035 if (RetVT.SimpleTy != MVT::v2f64)
13036 return Register();
13037 if ((Subtarget->isNeonAvailable())) {
13038 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13039 }
13040 return Register();
13041}
13042
13043Register fastEmit_ISD_STRICT_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13044 switch (VT.SimpleTy) {
13045 case MVT::f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
13046 case MVT::f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
13047 case MVT::f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
13048 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13049 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13050 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13051 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13052 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13053 default: return Register();
13054 }
13055}
13056
13057// FastEmit functions for ISD::STRICT_FMUL.
13058
13059Register fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13060 if (RetVT.SimpleTy != MVT::f16)
13061 return Register();
13062 if ((Subtarget->hasFullFP16())) {
13063 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13064 }
13065 return Register();
13066}
13067
13068Register fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13069 if (RetVT.SimpleTy != MVT::f32)
13070 return Register();
13071 if ((Subtarget->hasFPARMv8())) {
13072 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13073 }
13074 return Register();
13075}
13076
13077Register fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13078 if (RetVT.SimpleTy != MVT::f64)
13079 return Register();
13080 if ((Subtarget->hasFPARMv8())) {
13081 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13082 }
13083 return Register();
13084}
13085
13086Register fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13087 if (RetVT.SimpleTy != MVT::v4f16)
13088 return Register();
13089 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13090 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13091 }
13092 return Register();
13093}
13094
13095Register fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13096 if (RetVT.SimpleTy != MVT::v8f16)
13097 return Register();
13098 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13099 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13100 }
13101 return Register();
13102}
13103
13104Register fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13105 if (RetVT.SimpleTy != MVT::v2f32)
13106 return Register();
13107 if ((Subtarget->isNeonAvailable())) {
13108 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13109 }
13110 return Register();
13111}
13112
13113Register fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13114 if (RetVT.SimpleTy != MVT::v4f32)
13115 return Register();
13116 if ((Subtarget->isNeonAvailable())) {
13117 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13118 }
13119 return Register();
13120}
13121
13122Register fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13123 if (RetVT.SimpleTy != MVT::v2f64)
13124 return Register();
13125 if ((Subtarget->isNeonAvailable())) {
13126 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13127 }
13128 return Register();
13129}
13130
13131Register fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13132 switch (VT.SimpleTy) {
13133 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
13134 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
13135 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
13136 case MVT::v4f16: return fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
13137 case MVT::v8f16: return fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
13138 case MVT::v2f32: return fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
13139 case MVT::v4f32: return fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
13140 case MVT::v2f64: return fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
13141 default: return Register();
13142 }
13143}
13144
13145// FastEmit functions for ISD::STRICT_FSUB.
13146
13147Register fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13148 if (RetVT.SimpleTy != MVT::f16)
13149 return Register();
13150 if ((Subtarget->hasFullFP16())) {
13151 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13152 }
13153 return Register();
13154}
13155
13156Register fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13157 if (RetVT.SimpleTy != MVT::f32)
13158 return Register();
13159 if ((Subtarget->hasFPARMv8())) {
13160 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13161 }
13162 return Register();
13163}
13164
13165Register fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13166 if (RetVT.SimpleTy != MVT::f64)
13167 return Register();
13168 if ((Subtarget->hasFPARMv8())) {
13169 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13170 }
13171 return Register();
13172}
13173
13174Register fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13175 if (RetVT.SimpleTy != MVT::v4f16)
13176 return Register();
13177 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13178 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13179 }
13180 return Register();
13181}
13182
13183Register fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13184 if (RetVT.SimpleTy != MVT::v8f16)
13185 return Register();
13186 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13187 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13188 }
13189 return Register();
13190}
13191
13192Register fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13193 if (RetVT.SimpleTy != MVT::v2f32)
13194 return Register();
13195 if ((Subtarget->isNeonAvailable())) {
13196 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13197 }
13198 return Register();
13199}
13200
13201Register fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13202 if (RetVT.SimpleTy != MVT::v4f32)
13203 return Register();
13204 if ((Subtarget->isNeonAvailable())) {
13205 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13206 }
13207 return Register();
13208}
13209
13210Register fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13211 if (RetVT.SimpleTy != MVT::v2f64)
13212 return Register();
13213 if ((Subtarget->isNeonAvailable())) {
13214 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13215 }
13216 return Register();
13217}
13218
13219Register fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13220 switch (VT.SimpleTy) {
13221 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
13222 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
13223 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
13224 case MVT::v4f16: return fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
13225 case MVT::v8f16: return fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
13226 case MVT::v2f32: return fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
13227 case MVT::v4f32: return fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
13228 case MVT::v2f64: return fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
13229 default: return Register();
13230 }
13231}
13232
13233// FastEmit functions for ISD::SUB.
13234
13235Register fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13236 if (RetVT.SimpleTy != MVT::i32)
13237 return Register();
13238 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBSWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13239}
13240
13241Register fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13242 if (RetVT.SimpleTy != MVT::i64)
13243 return Register();
13244 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBSXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13245}
13246
13247Register fastEmit_ISD_SUB_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13248 if (RetVT.SimpleTy != MVT::v8i8)
13249 return Register();
13250 if ((Subtarget->isNeonAvailable())) {
13251 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13252 }
13253 return Register();
13254}
13255
13256Register fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13257 if (RetVT.SimpleTy != MVT::v16i8)
13258 return Register();
13259 if ((Subtarget->isNeonAvailable())) {
13260 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13261 }
13262 return Register();
13263}
13264
13265Register fastEmit_ISD_SUB_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13266 if (RetVT.SimpleTy != MVT::v4i16)
13267 return Register();
13268 if ((Subtarget->isNeonAvailable())) {
13269 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13270 }
13271 return Register();
13272}
13273
13274Register fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13275 if (RetVT.SimpleTy != MVT::v8i16)
13276 return Register();
13277 if ((Subtarget->isNeonAvailable())) {
13278 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13279 }
13280 return Register();
13281}
13282
13283Register fastEmit_ISD_SUB_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13284 if (RetVT.SimpleTy != MVT::v2i32)
13285 return Register();
13286 if ((Subtarget->isNeonAvailable())) {
13287 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13288 }
13289 return Register();
13290}
13291
13292Register fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13293 if (RetVT.SimpleTy != MVT::v4i32)
13294 return Register();
13295 if ((Subtarget->isNeonAvailable())) {
13296 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13297 }
13298 return Register();
13299}
13300
13301Register fastEmit_ISD_SUB_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13302 if (RetVT.SimpleTy != MVT::v1i64)
13303 return Register();
13304 if ((Subtarget->isNeonAvailable())) {
13305 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13306 }
13307 return Register();
13308}
13309
13310Register fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13311 if (RetVT.SimpleTy != MVT::v2i64)
13312 return Register();
13313 if ((Subtarget->isNeonAvailable())) {
13314 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13315 }
13316 return Register();
13317}
13318
13319Register fastEmit_ISD_SUB_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13320 if (RetVT.SimpleTy != MVT::nxv16i8)
13321 return Register();
13322 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13323 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13324 }
13325 return Register();
13326}
13327
13328Register fastEmit_ISD_SUB_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13329 if (RetVT.SimpleTy != MVT::nxv8i16)
13330 return Register();
13331 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13332 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13333 }
13334 return Register();
13335}
13336
13337Register fastEmit_ISD_SUB_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13338 if (RetVT.SimpleTy != MVT::nxv4i32)
13339 return Register();
13340 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13341 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13342 }
13343 return Register();
13344}
13345
13346Register fastEmit_ISD_SUB_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13347 if (RetVT.SimpleTy != MVT::nxv2i64)
13348 return Register();
13349 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13350 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13351 }
13352 return Register();
13353}
13354
13355Register fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13356 switch (VT.SimpleTy) {
13357 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
13358 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op1);
13359 case MVT::v8i8: return fastEmit_ISD_SUB_MVT_v8i8_rr(RetVT, Op0, Op1);
13360 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
13361 case MVT::v4i16: return fastEmit_ISD_SUB_MVT_v4i16_rr(RetVT, Op0, Op1);
13362 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
13363 case MVT::v2i32: return fastEmit_ISD_SUB_MVT_v2i32_rr(RetVT, Op0, Op1);
13364 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
13365 case MVT::v1i64: return fastEmit_ISD_SUB_MVT_v1i64_rr(RetVT, Op0, Op1);
13366 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
13367 case MVT::nxv16i8: return fastEmit_ISD_SUB_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13368 case MVT::nxv8i16: return fastEmit_ISD_SUB_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13369 case MVT::nxv4i32: return fastEmit_ISD_SUB_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13370 case MVT::nxv2i64: return fastEmit_ISD_SUB_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13371 default: return Register();
13372 }
13373}
13374
13375// FastEmit functions for ISD::UADDSAT.
13376
13377Register fastEmit_ISD_UADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13378 if (RetVT.SimpleTy != MVT::v8i8)
13379 return Register();
13380 if ((Subtarget->isNeonAvailable())) {
13381 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13382 }
13383 return Register();
13384}
13385
13386Register fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13387 if (RetVT.SimpleTy != MVT::v16i8)
13388 return Register();
13389 if ((Subtarget->isNeonAvailable())) {
13390 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13391 }
13392 return Register();
13393}
13394
13395Register fastEmit_ISD_UADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13396 if (RetVT.SimpleTy != MVT::v4i16)
13397 return Register();
13398 if ((Subtarget->isNeonAvailable())) {
13399 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13400 }
13401 return Register();
13402}
13403
13404Register fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13405 if (RetVT.SimpleTy != MVT::v8i16)
13406 return Register();
13407 if ((Subtarget->isNeonAvailable())) {
13408 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13409 }
13410 return Register();
13411}
13412
13413Register fastEmit_ISD_UADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13414 if (RetVT.SimpleTy != MVT::v2i32)
13415 return Register();
13416 if ((Subtarget->isNeonAvailable())) {
13417 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13418 }
13419 return Register();
13420}
13421
13422Register fastEmit_ISD_UADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13423 if (RetVT.SimpleTy != MVT::v4i32)
13424 return Register();
13425 if ((Subtarget->isNeonAvailable())) {
13426 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13427 }
13428 return Register();
13429}
13430
13431Register fastEmit_ISD_UADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13432 if (RetVT.SimpleTy != MVT::v1i64)
13433 return Register();
13434 if ((Subtarget->isNeonAvailable())) {
13435 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13436 }
13437 return Register();
13438}
13439
13440Register fastEmit_ISD_UADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13441 if (RetVT.SimpleTy != MVT::v2i64)
13442 return Register();
13443 if ((Subtarget->isNeonAvailable())) {
13444 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13445 }
13446 return Register();
13447}
13448
13449Register fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13450 if (RetVT.SimpleTy != MVT::nxv16i8)
13451 return Register();
13452 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13453 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13454 }
13455 return Register();
13456}
13457
13458Register fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13459 if (RetVT.SimpleTy != MVT::nxv8i16)
13460 return Register();
13461 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13462 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13463 }
13464 return Register();
13465}
13466
13467Register fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13468 if (RetVT.SimpleTy != MVT::nxv4i32)
13469 return Register();
13470 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13471 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13472 }
13473 return Register();
13474}
13475
13476Register fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13477 if (RetVT.SimpleTy != MVT::nxv2i64)
13478 return Register();
13479 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13480 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13481 }
13482 return Register();
13483}
13484
13485Register fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13486 switch (VT.SimpleTy) {
13487 case MVT::v8i8: return fastEmit_ISD_UADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
13488 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13489 case MVT::v4i16: return fastEmit_ISD_UADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
13490 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13491 case MVT::v2i32: return fastEmit_ISD_UADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
13492 case MVT::v4i32: return fastEmit_ISD_UADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
13493 case MVT::v1i64: return fastEmit_ISD_UADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
13494 case MVT::v2i64: return fastEmit_ISD_UADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
13495 case MVT::nxv16i8: return fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13496 case MVT::nxv8i16: return fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13497 case MVT::nxv4i32: return fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13498 case MVT::nxv2i64: return fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13499 default: return Register();
13500 }
13501}
13502
13503// FastEmit functions for ISD::UDIV.
13504
13505Register fastEmit_ISD_UDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13506 if (RetVT.SimpleTy != MVT::i32)
13507 return Register();
13508 return fastEmitInst_rr(MachineInstOpcode: AArch64::UDIVWr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13509}
13510
13511Register fastEmit_ISD_UDIV_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13512 if (RetVT.SimpleTy != MVT::i64)
13513 return Register();
13514 return fastEmitInst_rr(MachineInstOpcode: AArch64::UDIVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13515}
13516
13517Register fastEmit_ISD_UDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13518 switch (VT.SimpleTy) {
13519 case MVT::i32: return fastEmit_ISD_UDIV_MVT_i32_rr(RetVT, Op0, Op1);
13520 case MVT::i64: return fastEmit_ISD_UDIV_MVT_i64_rr(RetVT, Op0, Op1);
13521 default: return Register();
13522 }
13523}
13524
13525// FastEmit functions for ISD::UMAX.
13526
13527Register fastEmit_ISD_UMAX_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13528 if (RetVT.SimpleTy != MVT::i32)
13529 return Register();
13530 if ((Subtarget->hasCSSC())) {
13531 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13532 }
13533 return Register();
13534}
13535
13536Register fastEmit_ISD_UMAX_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13537 if (RetVT.SimpleTy != MVT::i64)
13538 return Register();
13539 if ((Subtarget->hasCSSC())) {
13540 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13541 }
13542 return Register();
13543}
13544
13545Register fastEmit_ISD_UMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13546 if (RetVT.SimpleTy != MVT::v8i8)
13547 return Register();
13548 if ((Subtarget->isNeonAvailable())) {
13549 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13550 }
13551 return Register();
13552}
13553
13554Register fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13555 if (RetVT.SimpleTy != MVT::v16i8)
13556 return Register();
13557 if ((Subtarget->isNeonAvailable())) {
13558 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13559 }
13560 return Register();
13561}
13562
13563Register fastEmit_ISD_UMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13564 if (RetVT.SimpleTy != MVT::v4i16)
13565 return Register();
13566 if ((Subtarget->isNeonAvailable())) {
13567 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13568 }
13569 return Register();
13570}
13571
13572Register fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13573 if (RetVT.SimpleTy != MVT::v8i16)
13574 return Register();
13575 if ((Subtarget->isNeonAvailable())) {
13576 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13577 }
13578 return Register();
13579}
13580
13581Register fastEmit_ISD_UMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13582 if (RetVT.SimpleTy != MVT::v2i32)
13583 return Register();
13584 if ((Subtarget->isNeonAvailable())) {
13585 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13586 }
13587 return Register();
13588}
13589
13590Register fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13591 if (RetVT.SimpleTy != MVT::v4i32)
13592 return Register();
13593 if ((Subtarget->isNeonAvailable())) {
13594 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13595 }
13596 return Register();
13597}
13598
13599Register fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13600 switch (VT.SimpleTy) {
13601 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_rr(RetVT, Op0, Op1);
13602 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_rr(RetVT, Op0, Op1);
13603 case MVT::v8i8: return fastEmit_ISD_UMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
13604 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
13605 case MVT::v4i16: return fastEmit_ISD_UMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
13606 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
13607 case MVT::v2i32: return fastEmit_ISD_UMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
13608 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
13609 default: return Register();
13610 }
13611}
13612
13613// FastEmit functions for ISD::UMIN.
13614
13615Register fastEmit_ISD_UMIN_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13616 if (RetVT.SimpleTy != MVT::i32)
13617 return Register();
13618 if ((Subtarget->hasCSSC())) {
13619 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13620 }
13621 return Register();
13622}
13623
13624Register fastEmit_ISD_UMIN_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13625 if (RetVT.SimpleTy != MVT::i64)
13626 return Register();
13627 if ((Subtarget->hasCSSC())) {
13628 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13629 }
13630 return Register();
13631}
13632
13633Register fastEmit_ISD_UMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13634 if (RetVT.SimpleTy != MVT::v8i8)
13635 return Register();
13636 if ((Subtarget->isNeonAvailable())) {
13637 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13638 }
13639 return Register();
13640}
13641
13642Register fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13643 if (RetVT.SimpleTy != MVT::v16i8)
13644 return Register();
13645 if ((Subtarget->isNeonAvailable())) {
13646 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13647 }
13648 return Register();
13649}
13650
13651Register fastEmit_ISD_UMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13652 if (RetVT.SimpleTy != MVT::v4i16)
13653 return Register();
13654 if ((Subtarget->isNeonAvailable())) {
13655 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13656 }
13657 return Register();
13658}
13659
13660Register fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13661 if (RetVT.SimpleTy != MVT::v8i16)
13662 return Register();
13663 if ((Subtarget->isNeonAvailable())) {
13664 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13665 }
13666 return Register();
13667}
13668
13669Register fastEmit_ISD_UMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13670 if (RetVT.SimpleTy != MVT::v2i32)
13671 return Register();
13672 if ((Subtarget->isNeonAvailable())) {
13673 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13674 }
13675 return Register();
13676}
13677
13678Register fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13679 if (RetVT.SimpleTy != MVT::v4i32)
13680 return Register();
13681 if ((Subtarget->isNeonAvailable())) {
13682 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13683 }
13684 return Register();
13685}
13686
13687Register fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13688 switch (VT.SimpleTy) {
13689 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_rr(RetVT, Op0, Op1);
13690 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_rr(RetVT, Op0, Op1);
13691 case MVT::v8i8: return fastEmit_ISD_UMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
13692 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
13693 case MVT::v4i16: return fastEmit_ISD_UMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
13694 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
13695 case MVT::v2i32: return fastEmit_ISD_UMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
13696 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
13697 default: return Register();
13698 }
13699}
13700
13701// FastEmit functions for ISD::USUBSAT.
13702
13703Register fastEmit_ISD_USUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13704 if (RetVT.SimpleTy != MVT::v8i8)
13705 return Register();
13706 if ((Subtarget->isNeonAvailable())) {
13707 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13708 }
13709 return Register();
13710}
13711
13712Register fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13713 if (RetVT.SimpleTy != MVT::v16i8)
13714 return Register();
13715 if ((Subtarget->isNeonAvailable())) {
13716 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13717 }
13718 return Register();
13719}
13720
13721Register fastEmit_ISD_USUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13722 if (RetVT.SimpleTy != MVT::v4i16)
13723 return Register();
13724 if ((Subtarget->isNeonAvailable())) {
13725 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13726 }
13727 return Register();
13728}
13729
13730Register fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13731 if (RetVT.SimpleTy != MVT::v8i16)
13732 return Register();
13733 if ((Subtarget->isNeonAvailable())) {
13734 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13735 }
13736 return Register();
13737}
13738
13739Register fastEmit_ISD_USUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13740 if (RetVT.SimpleTy != MVT::v2i32)
13741 return Register();
13742 if ((Subtarget->isNeonAvailable())) {
13743 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13744 }
13745 return Register();
13746}
13747
13748Register fastEmit_ISD_USUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13749 if (RetVT.SimpleTy != MVT::v4i32)
13750 return Register();
13751 if ((Subtarget->isNeonAvailable())) {
13752 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13753 }
13754 return Register();
13755}
13756
13757Register fastEmit_ISD_USUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13758 if (RetVT.SimpleTy != MVT::v1i64)
13759 return Register();
13760 if ((Subtarget->isNeonAvailable())) {
13761 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13762 }
13763 return Register();
13764}
13765
13766Register fastEmit_ISD_USUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13767 if (RetVT.SimpleTy != MVT::v2i64)
13768 return Register();
13769 if ((Subtarget->isNeonAvailable())) {
13770 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13771 }
13772 return Register();
13773}
13774
13775Register fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13776 if (RetVT.SimpleTy != MVT::nxv16i8)
13777 return Register();
13778 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13779 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13780 }
13781 return Register();
13782}
13783
13784Register fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13785 if (RetVT.SimpleTy != MVT::nxv8i16)
13786 return Register();
13787 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13788 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13789 }
13790 return Register();
13791}
13792
13793Register fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13794 if (RetVT.SimpleTy != MVT::nxv4i32)
13795 return Register();
13796 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13797 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13798 }
13799 return Register();
13800}
13801
13802Register fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13803 if (RetVT.SimpleTy != MVT::nxv2i64)
13804 return Register();
13805 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13806 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13807 }
13808 return Register();
13809}
13810
13811Register fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13812 switch (VT.SimpleTy) {
13813 case MVT::v8i8: return fastEmit_ISD_USUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
13814 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13815 case MVT::v4i16: return fastEmit_ISD_USUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
13816 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13817 case MVT::v2i32: return fastEmit_ISD_USUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
13818 case MVT::v4i32: return fastEmit_ISD_USUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
13819 case MVT::v1i64: return fastEmit_ISD_USUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
13820 case MVT::v2i64: return fastEmit_ISD_USUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
13821 case MVT::nxv16i8: return fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13822 case MVT::nxv8i16: return fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13823 case MVT::nxv4i32: return fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13824 case MVT::nxv2i64: return fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13825 default: return Register();
13826 }
13827}
13828
13829// FastEmit functions for ISD::XOR.
13830
13831Register fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13832 if (RetVT.SimpleTy != MVT::i32)
13833 return Register();
13834 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13835}
13836
13837Register fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13838 if (RetVT.SimpleTy != MVT::i64)
13839 return Register();
13840 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13841}
13842
13843Register fastEmit_ISD_XOR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13844 if (RetVT.SimpleTy != MVT::v8i8)
13845 return Register();
13846 if ((Subtarget->isNeonAvailable())) {
13847 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13848 }
13849 return Register();
13850}
13851
13852Register fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13853 if (RetVT.SimpleTy != MVT::v16i8)
13854 return Register();
13855 if ((Subtarget->isNeonAvailable())) {
13856 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13857 }
13858 return Register();
13859}
13860
13861Register fastEmit_ISD_XOR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13862 if (RetVT.SimpleTy != MVT::v4i16)
13863 return Register();
13864 if ((Subtarget->isNeonAvailable())) {
13865 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13866 }
13867 return Register();
13868}
13869
13870Register fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13871 if (RetVT.SimpleTy != MVT::v8i16)
13872 return Register();
13873 if ((Subtarget->isNeonAvailable())) {
13874 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13875 }
13876 return Register();
13877}
13878
13879Register fastEmit_ISD_XOR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13880 if (RetVT.SimpleTy != MVT::v2i32)
13881 return Register();
13882 if ((Subtarget->isNeonAvailable())) {
13883 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13884 }
13885 return Register();
13886}
13887
13888Register fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13889 if (RetVT.SimpleTy != MVT::v4i32)
13890 return Register();
13891 if ((Subtarget->isNeonAvailable())) {
13892 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13893 }
13894 return Register();
13895}
13896
13897Register fastEmit_ISD_XOR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13898 if (RetVT.SimpleTy != MVT::v1i64)
13899 return Register();
13900 if ((Subtarget->isNeonAvailable())) {
13901 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13902 }
13903 return Register();
13904}
13905
13906Register fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13907 if (RetVT.SimpleTy != MVT::v2i64)
13908 return Register();
13909 if ((Subtarget->isNeonAvailable())) {
13910 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13911 }
13912 return Register();
13913}
13914
13915Register fastEmit_ISD_XOR_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13916 if (RetVT.SimpleTy != MVT::nxv16i8)
13917 return Register();
13918 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13919 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
13920 }
13921 return Register();
13922}
13923
13924Register fastEmit_ISD_XOR_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13925 if (RetVT.SimpleTy != MVT::nxv8i16)
13926 return Register();
13927 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13928 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
13929 }
13930 return Register();
13931}
13932
13933Register fastEmit_ISD_XOR_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13934 if (RetVT.SimpleTy != MVT::nxv4i32)
13935 return Register();
13936 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13937 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
13938 }
13939 return Register();
13940}
13941
13942Register fastEmit_ISD_XOR_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13943 if (RetVT.SimpleTy != MVT::nxv2i64)
13944 return Register();
13945 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13946 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
13947 }
13948 return Register();
13949}
13950
13951Register fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13952 switch (VT.SimpleTy) {
13953 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
13954 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op1);
13955 case MVT::v8i8: return fastEmit_ISD_XOR_MVT_v8i8_rr(RetVT, Op0, Op1);
13956 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
13957 case MVT::v4i16: return fastEmit_ISD_XOR_MVT_v4i16_rr(RetVT, Op0, Op1);
13958 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
13959 case MVT::v2i32: return fastEmit_ISD_XOR_MVT_v2i32_rr(RetVT, Op0, Op1);
13960 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
13961 case MVT::v1i64: return fastEmit_ISD_XOR_MVT_v1i64_rr(RetVT, Op0, Op1);
13962 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
13963 case MVT::nxv16i8: return fastEmit_ISD_XOR_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13964 case MVT::nxv8i16: return fastEmit_ISD_XOR_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13965 case MVT::nxv4i32: return fastEmit_ISD_XOR_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13966 case MVT::nxv2i64: return fastEmit_ISD_XOR_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13967 default: return Register();
13968 }
13969}
13970
13971// Top-level FastEmit function.
13972
13973Register fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, Register Op1) override {
13974 switch (Opcode) {
13975 case AArch64ISD::ADDHN: return fastEmit_AArch64ISD_ADDHN_rr(VT, RetVT, Op0, Op1);
13976 case AArch64ISD::ADDP: return fastEmit_AArch64ISD_ADDP_rr(VT, RetVT, Op0, Op1);
13977 case AArch64ISD::BIC: return fastEmit_AArch64ISD_BIC_rr(VT, RetVT, Op0, Op1);
13978 case AArch64ISD::CMTST: return fastEmit_AArch64ISD_CMTST_rr(VT, RetVT, Op0, Op1);
13979 case AArch64ISD::CTTZ_ELTS_ZERO_POISON: return fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_rr(VT, RetVT, Op0, Op1);
13980 case AArch64ISD::FCMEQ: return fastEmit_AArch64ISD_FCMEQ_rr(VT, RetVT, Op0, Op1);
13981 case AArch64ISD::FCMGE: return fastEmit_AArch64ISD_FCMGE_rr(VT, RetVT, Op0, Op1);
13982 case AArch64ISD::FCMGT: return fastEmit_AArch64ISD_FCMGT_rr(VT, RetVT, Op0, Op1);
13983 case AArch64ISD::FRECPS: return fastEmit_AArch64ISD_FRECPS_rr(VT, RetVT, Op0, Op1);
13984 case AArch64ISD::FRSQRTS: return fastEmit_AArch64ISD_FRSQRTS_rr(VT, RetVT, Op0, Op1);
13985 case AArch64ISD::PMULL: return fastEmit_AArch64ISD_PMULL_rr(VT, RetVT, Op0, Op1);
13986 case AArch64ISD::SMULL: return fastEmit_AArch64ISD_SMULL_rr(VT, RetVT, Op0, Op1);
13987 case AArch64ISD::SQADD: return fastEmit_AArch64ISD_SQADD_rr(VT, RetVT, Op0, Op1);
13988 case AArch64ISD::SQDMULH: return fastEmit_AArch64ISD_SQDMULH_rr(VT, RetVT, Op0, Op1);
13989 case AArch64ISD::SQDMULL: return fastEmit_AArch64ISD_SQDMULL_rr(VT, RetVT, Op0, Op1);
13990 case AArch64ISD::SQRDMULH: return fastEmit_AArch64ISD_SQRDMULH_rr(VT, RetVT, Op0, Op1);
13991 case AArch64ISD::SQRSHL: return fastEmit_AArch64ISD_SQRSHL_rr(VT, RetVT, Op0, Op1);
13992 case AArch64ISD::SQSHL: return fastEmit_AArch64ISD_SQSHL_rr(VT, RetVT, Op0, Op1);
13993 case AArch64ISD::SQSUB: return fastEmit_AArch64ISD_SQSUB_rr(VT, RetVT, Op0, Op1);
13994 case AArch64ISD::SUQADD: return fastEmit_AArch64ISD_SUQADD_rr(VT, RetVT, Op0, Op1);
13995 case AArch64ISD::TBL: return fastEmit_AArch64ISD_TBL_rr(VT, RetVT, Op0, Op1);
13996 case AArch64ISD::TRN1: return fastEmit_AArch64ISD_TRN1_rr(VT, RetVT, Op0, Op1);
13997 case AArch64ISD::TRN2: return fastEmit_AArch64ISD_TRN2_rr(VT, RetVT, Op0, Op1);
13998 case AArch64ISD::UMULL: return fastEmit_AArch64ISD_UMULL_rr(VT, RetVT, Op0, Op1);
13999 case AArch64ISD::UQADD: return fastEmit_AArch64ISD_UQADD_rr(VT, RetVT, Op0, Op1);
14000 case AArch64ISD::UQRSHL: return fastEmit_AArch64ISD_UQRSHL_rr(VT, RetVT, Op0, Op1);
14001 case AArch64ISD::UQSHL: return fastEmit_AArch64ISD_UQSHL_rr(VT, RetVT, Op0, Op1);
14002 case AArch64ISD::UQSUB: return fastEmit_AArch64ISD_UQSUB_rr(VT, RetVT, Op0, Op1);
14003 case AArch64ISD::USQADD: return fastEmit_AArch64ISD_USQADD_rr(VT, RetVT, Op0, Op1);
14004 case AArch64ISD::UZP1: return fastEmit_AArch64ISD_UZP1_rr(VT, RetVT, Op0, Op1);
14005 case AArch64ISD::UZP2: return fastEmit_AArch64ISD_UZP2_rr(VT, RetVT, Op0, Op1);
14006 case AArch64ISD::ZIP1: return fastEmit_AArch64ISD_ZIP1_rr(VT, RetVT, Op0, Op1);
14007 case AArch64ISD::ZIP2: return fastEmit_AArch64ISD_ZIP2_rr(VT, RetVT, Op0, Op1);
14008 case ISD::ABDS: return fastEmit_ISD_ABDS_rr(VT, RetVT, Op0, Op1);
14009 case ISD::ABDU: return fastEmit_ISD_ABDU_rr(VT, RetVT, Op0, Op1);
14010 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
14011 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
14012 case ISD::AVGCEILS: return fastEmit_ISD_AVGCEILS_rr(VT, RetVT, Op0, Op1);
14013 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
14014 case ISD::AVGFLOORS: return fastEmit_ISD_AVGFLOORS_rr(VT, RetVT, Op0, Op1);
14015 case ISD::AVGFLOORU: return fastEmit_ISD_AVGFLOORU_rr(VT, RetVT, Op0, Op1);
14016 case ISD::CLMUL: return fastEmit_ISD_CLMUL_rr(VT, RetVT, Op0, Op1);
14017 case ISD::CONCAT_VECTORS: return fastEmit_ISD_CONCAT_VECTORS_rr(VT, RetVT, Op0, Op1);
14018 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
14019 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
14020 case ISD::FMAXIMUM: return fastEmit_ISD_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
14021 case ISD::FMAXNUM: return fastEmit_ISD_FMAXNUM_rr(VT, RetVT, Op0, Op1);
14022 case ISD::FMAXNUM_IEEE: return fastEmit_ISD_FMAXNUM_IEEE_rr(VT, RetVT, Op0, Op1);
14023 case ISD::FMINIMUM: return fastEmit_ISD_FMINIMUM_rr(VT, RetVT, Op0, Op1);
14024 case ISD::FMINNUM: return fastEmit_ISD_FMINNUM_rr(VT, RetVT, Op0, Op1);
14025 case ISD::FMINNUM_IEEE: return fastEmit_ISD_FMINNUM_IEEE_rr(VT, RetVT, Op0, Op1);
14026 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
14027 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
14028 case ISD::GET_ACTIVE_LANE_MASK: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_rr(VT, RetVT, Op0, Op1);
14029 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
14030 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
14031 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
14032 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
14033 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
14034 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
14035 case ISD::SDIV: return fastEmit_ISD_SDIV_rr(VT, RetVT, Op0, Op1);
14036 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
14037 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
14038 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
14039 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
14040 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
14041 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
14042 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1);
14043 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1);
14044 case ISD::STRICT_FMAXIMUM: return fastEmit_ISD_STRICT_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
14045 case ISD::STRICT_FMAXNUM: return fastEmit_ISD_STRICT_FMAXNUM_rr(VT, RetVT, Op0, Op1);
14046 case ISD::STRICT_FMINIMUM: return fastEmit_ISD_STRICT_FMINIMUM_rr(VT, RetVT, Op0, Op1);
14047 case ISD::STRICT_FMINNUM: return fastEmit_ISD_STRICT_FMINNUM_rr(VT, RetVT, Op0, Op1);
14048 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1);
14049 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1);
14050 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
14051 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
14052 case ISD::UDIV: return fastEmit_ISD_UDIV_rr(VT, RetVT, Op0, Op1);
14053 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
14054 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
14055 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
14056 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
14057 default: return Register();
14058 }
14059}
14060
14061// FastEmit functions for AArch64ISD::DUPLANE64.
14062
14063Register fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14064 if (RetVT.SimpleTy != MVT::v2i64)
14065 return Register();
14066 if ((Subtarget->isNeonAvailable())) {
14067 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i64lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14068 }
14069 return Register();
14070}
14071
14072Register fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14073 if (RetVT.SimpleTy != MVT::v2f64)
14074 return Register();
14075 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i64lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14076}
14077
14078Register fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14079 switch (VT.SimpleTy) {
14080 case MVT::v2i64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14081 case MVT::v2f64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14082 default: return Register();
14083 }
14084}
14085
14086// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14087
14088Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14089 if (RetVT.SimpleTy != MVT::i64)
14090 return Register();
14091 if ((Subtarget->isNeonAvailable())) {
14092 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi64, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14093 }
14094 return Register();
14095}
14096
14097Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14098 if (RetVT.SimpleTy != MVT::f64)
14099 return Register();
14100 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi64, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14101}
14102
14103Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14104 switch (VT.SimpleTy) {
14105 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14106 case MVT::v2f64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14107 default: return Register();
14108 }
14109}
14110
14111// Top-level FastEmit function.
14112
14113Register fastEmit_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14114 switch (Opcode) {
14115 case AArch64ISD::DUPLANE64: return fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1);
14116 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1);
14117 default: return Register();
14118 }
14119}
14120
14121// FastEmit functions for AArch64ISD::DUPLANE32.
14122
14123Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14124 if ((Subtarget->isNeonAvailable())) {
14125 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i32lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14126 }
14127 return Register();
14128}
14129
14130Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14131 if ((Subtarget->isNeonAvailable())) {
14132 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i32lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14133 }
14134 return Register();
14135}
14136
14137Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14138switch (RetVT.SimpleTy) {
14139 case MVT::v2i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Op0, imm1);
14140 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Op0, imm1);
14141 default: return Register();
14142}
14143}
14144
14145Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14146 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i32lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14147}
14148
14149Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14150 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i32lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14151}
14152
14153Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14154switch (RetVT.SimpleTy) {
14155 case MVT::v2f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Op0, imm1);
14156 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Op0, imm1);
14157 default: return Register();
14158}
14159}
14160
14161Register fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14162 switch (VT.SimpleTy) {
14163 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14164 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14165 default: return Register();
14166 }
14167}
14168
14169// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14170
14171Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14172 if (RetVT.SimpleTy != MVT::i32)
14173 return Register();
14174 if ((Subtarget->isNeonAvailable())) {
14175 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi32, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14176 }
14177 return Register();
14178}
14179
14180Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14181 if (RetVT.SimpleTy != MVT::f32)
14182 return Register();
14183 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi32, RC: &AArch64::FPR32RegClass, Op0, Imm: imm1);
14184}
14185
14186Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14187 switch (VT.SimpleTy) {
14188 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14189 case MVT::v4f32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14190 default: return Register();
14191 }
14192}
14193
14194// Top-level FastEmit function.
14195
14196Register fastEmit_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14197 switch (Opcode) {
14198 case AArch64ISD::DUPLANE32: return fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1);
14199 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1);
14200 default: return Register();
14201 }
14202}
14203
14204// FastEmit functions for AArch64ISD::DUPLANE16.
14205
14206Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14207 if ((Subtarget->isNeonAvailable())) {
14208 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14209 }
14210 return Register();
14211}
14212
14213Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14214 if ((Subtarget->isNeonAvailable())) {
14215 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14216 }
14217 return Register();
14218}
14219
14220Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14221switch (RetVT.SimpleTy) {
14222 case MVT::v4i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Op0, imm1);
14223 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Op0, imm1);
14224 default: return Register();
14225}
14226}
14227
14228Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14229 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14230}
14231
14232Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14233 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14234}
14235
14236Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14237switch (RetVT.SimpleTy) {
14238 case MVT::v4f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Op0, imm1);
14239 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Op0, imm1);
14240 default: return Register();
14241}
14242}
14243
14244Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14245 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14246}
14247
14248Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14249 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14250}
14251
14252Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14253switch (RetVT.SimpleTy) {
14254 case MVT::v4bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(Op0, imm1);
14255 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(Op0, imm1);
14256 default: return Register();
14257}
14258}
14259
14260Register fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14261 switch (VT.SimpleTy) {
14262 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14263 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14264 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14265 default: return Register();
14266 }
14267}
14268
14269// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14270
14271Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14272 if (RetVT.SimpleTy != MVT::i32)
14273 return Register();
14274 if ((Subtarget->isNeonAvailable())) {
14275 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi16, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14276 }
14277 return Register();
14278}
14279
14280Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14281 if (RetVT.SimpleTy != MVT::f16)
14282 return Register();
14283 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi16, RC: &AArch64::FPR16RegClass, Op0, Imm: imm1);
14284}
14285
14286Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14287 if (RetVT.SimpleTy != MVT::bf16)
14288 return Register();
14289 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi16, RC: &AArch64::FPR16RegClass, Op0, Imm: imm1);
14290}
14291
14292Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14293 switch (VT.SimpleTy) {
14294 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14295 case MVT::v8f16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14296 case MVT::v8bf16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14297 default: return Register();
14298 }
14299}
14300
14301// Top-level FastEmit function.
14302
14303Register fastEmit_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14304 switch (Opcode) {
14305 case AArch64ISD::DUPLANE16: return fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1);
14306 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1);
14307 default: return Register();
14308 }
14309}
14310
14311// FastEmit functions for AArch64ISD::DUPLANE8.
14312
14313Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Register Op0, uint64_t imm1) {
14314 if ((Subtarget->isNeonAvailable())) {
14315 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i8lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14316 }
14317 return Register();
14318}
14319
14320Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Register Op0, uint64_t imm1) {
14321 if ((Subtarget->isNeonAvailable())) {
14322 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv16i8lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14323 }
14324 return Register();
14325}
14326
14327Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, Register Op0, uint64_t imm1) {
14328switch (RetVT.SimpleTy) {
14329 case MVT::v8i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Op0, imm1);
14330 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Op0, imm1);
14331 default: return Register();
14332}
14333}
14334
14335Register fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14336 switch (VT.SimpleTy) {
14337 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1);
14338 default: return Register();
14339 }
14340}
14341
14342// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14343
14344Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, Register Op0, uint64_t imm1) {
14345 if (RetVT.SimpleTy != MVT::i32)
14346 return Register();
14347 if ((Subtarget->isNeonAvailable())) {
14348 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi8, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14349 }
14350 return Register();
14351}
14352
14353Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14354 switch (VT.SimpleTy) {
14355 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1);
14356 default: return Register();
14357 }
14358}
14359
14360// Top-level FastEmit function.
14361
14362Register fastEmit_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14363 switch (Opcode) {
14364 case AArch64ISD::DUPLANE8: return fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1);
14365 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1);
14366 default: return Register();
14367 }
14368}
14369
14370// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14371
14372Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14373 if (RetVT.SimpleTy != MVT::i32)
14374 return Register();
14375 if ((Subtarget->hasNEON())) {
14376 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi8_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14377 }
14378 return Register();
14379}
14380
14381Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14382 if (RetVT.SimpleTy != MVT::i32)
14383 return Register();
14384 if ((Subtarget->hasNEON())) {
14385 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi16_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14386 }
14387 return Register();
14388}
14389
14390Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14391 if (RetVT.SimpleTy != MVT::i32)
14392 return Register();
14393 if ((Subtarget->hasNEON())) {
14394 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi32_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14395 }
14396 return Register();
14397}
14398
14399Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14400 if (RetVT.SimpleTy != MVT::i64)
14401 return Register();
14402 if ((Subtarget->hasNEON())) {
14403 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi64_idx0, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14404 }
14405 return Register();
14406}
14407
14408Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14409 switch (VT.SimpleTy) {
14410 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14411 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14412 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14413 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14414 default: return Register();
14415 }
14416}
14417
14418// Top-level FastEmit function.
14419
14420Register fastEmit_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14421 switch (Opcode) {
14422 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(VT, RetVT, Op0, imm1);
14423 default: return Register();
14424 }
14425}
14426
14427// FastEmit functions for ISD::SMAX.
14428
14429Register fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14430 if (RetVT.SimpleTy != MVT::i32)
14431 return Register();
14432 if ((Subtarget->hasCSSC())) {
14433 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMAXWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14434 }
14435 return Register();
14436}
14437
14438Register fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14439 switch (VT.SimpleTy) {
14440 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1);
14441 default: return Register();
14442 }
14443}
14444
14445// FastEmit functions for ISD::SMIN.
14446
14447Register fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14448 if (RetVT.SimpleTy != MVT::i32)
14449 return Register();
14450 if ((Subtarget->hasCSSC())) {
14451 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMINWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14452 }
14453 return Register();
14454}
14455
14456Register fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14457 switch (VT.SimpleTy) {
14458 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1);
14459 default: return Register();
14460 }
14461}
14462
14463// Top-level FastEmit function.
14464
14465Register fastEmit_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14466 switch (Opcode) {
14467 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1);
14468 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1);
14469 default: return Register();
14470 }
14471}
14472
14473// FastEmit functions for ISD::SMAX.
14474
14475Register fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14476 if (RetVT.SimpleTy != MVT::i64)
14477 return Register();
14478 if ((Subtarget->hasCSSC())) {
14479 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMAXXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14480 }
14481 return Register();
14482}
14483
14484Register fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14485 switch (VT.SimpleTy) {
14486 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1);
14487 default: return Register();
14488 }
14489}
14490
14491// FastEmit functions for ISD::SMIN.
14492
14493Register fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14494 if (RetVT.SimpleTy != MVT::i64)
14495 return Register();
14496 if ((Subtarget->hasCSSC())) {
14497 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMINXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14498 }
14499 return Register();
14500}
14501
14502Register fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14503 switch (VT.SimpleTy) {
14504 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1);
14505 default: return Register();
14506 }
14507}
14508
14509// Top-level FastEmit function.
14510
14511Register fastEmit_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14512 switch (Opcode) {
14513 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1);
14514 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1);
14515 default: return Register();
14516 }
14517}
14518
14519// FastEmit functions for ISD::UMAX.
14520
14521Register fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14522 if (RetVT.SimpleTy != MVT::i32)
14523 return Register();
14524 if ((Subtarget->hasCSSC())) {
14525 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMAXWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14526 }
14527 return Register();
14528}
14529
14530Register fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14531 switch (VT.SimpleTy) {
14532 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1);
14533 default: return Register();
14534 }
14535}
14536
14537// FastEmit functions for ISD::UMIN.
14538
14539Register fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14540 if (RetVT.SimpleTy != MVT::i32)
14541 return Register();
14542 if ((Subtarget->hasCSSC())) {
14543 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMINWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14544 }
14545 return Register();
14546}
14547
14548Register fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14549 switch (VT.SimpleTy) {
14550 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1);
14551 default: return Register();
14552 }
14553}
14554
14555// Top-level FastEmit function.
14556
14557Register fastEmit_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14558 switch (Opcode) {
14559 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1);
14560 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1);
14561 default: return Register();
14562 }
14563}
14564
14565// FastEmit functions for ISD::UMAX.
14566
14567Register fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14568 if (RetVT.SimpleTy != MVT::i64)
14569 return Register();
14570 if ((Subtarget->hasCSSC())) {
14571 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMAXXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14572 }
14573 return Register();
14574}
14575
14576Register fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14577 switch (VT.SimpleTy) {
14578 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1);
14579 default: return Register();
14580 }
14581}
14582
14583// FastEmit functions for ISD::UMIN.
14584
14585Register fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14586 if (RetVT.SimpleTy != MVT::i64)
14587 return Register();
14588 if ((Subtarget->hasCSSC())) {
14589 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMINXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14590 }
14591 return Register();
14592}
14593
14594Register fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14595 switch (VT.SimpleTy) {
14596 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1);
14597 default: return Register();
14598 }
14599}
14600
14601// Top-level FastEmit function.
14602
14603Register fastEmit_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14604 switch (Opcode) {
14605 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1);
14606 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1);
14607 default: return Register();
14608 }
14609}
14610
14611// FastEmit functions for ISD::Constant.
14612
14613Register fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
14614 if (RetVT.SimpleTy != MVT::i32)
14615 return Register();
14616 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVi32imm, RC: &AArch64::GPR32RegClass, Imm: imm0);
14617}
14618
14619Register fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) {
14620 if (RetVT.SimpleTy != MVT::i64)
14621 return Register();
14622 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVi64imm, RC: &AArch64::GPR64RegClass, Imm: imm0);
14623}
14624
14625Register fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
14626 switch (VT.SimpleTy) {
14627 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
14628 case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0);
14629 default: return Register();
14630 }
14631}
14632
14633// Top-level FastEmit function.
14634
14635Register fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
14636 if (VT == MVT::i32 && Predicate_imm0_255(Imm: imm0))
14637 if (Register Reg = fastEmit_i_Predicate_imm0_255(VT, RetVT, Opcode, imm0))
14638 return Reg;
14639
14640 if (VT == MVT::i32 && Predicate_simm6_32b(Imm: imm0))
14641 if (Register Reg = fastEmit_i_Predicate_simm6_32b(VT, RetVT, Opcode, imm0))
14642 return Reg;
14643
14644 switch (Opcode) {
14645 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
14646 default: return Register();
14647 }
14648}
14649
14650// FastEmit functions for AArch64ISD::FMOV.
14651
14652Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(uint64_t imm0) {
14653 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
14654 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv4f16_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
14655 }
14656 return Register();
14657}
14658
14659Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(uint64_t imm0) {
14660 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
14661 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv8f16_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14662 }
14663 return Register();
14664}
14665
14666Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(uint64_t imm0) {
14667 if ((Subtarget->isNeonAvailable())) {
14668 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv2f32_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
14669 }
14670 return Register();
14671}
14672
14673Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(uint64_t imm0) {
14674 if ((Subtarget->isNeonAvailable())) {
14675 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv4f32_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14676 }
14677 return Register();
14678}
14679
14680Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(uint64_t imm0) {
14681 if ((Subtarget->isNeonAvailable())) {
14682 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv2f64_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14683 }
14684 return Register();
14685}
14686
14687Register fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
14688switch (RetVT.SimpleTy) {
14689 case MVT::v4f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(imm0);
14690 case MVT::v8f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(imm0);
14691 case MVT::v2f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(imm0);
14692 case MVT::v4f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(imm0);
14693 case MVT::v2f64: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(imm0);
14694 default: return Register();
14695}
14696}
14697
14698Register fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
14699 switch (VT.SimpleTy) {
14700 case MVT::i32: return fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
14701 default: return Register();
14702 }
14703}
14704
14705// FastEmit functions for AArch64ISD::MOVI.
14706
14707Register fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(uint64_t imm0) {
14708 if ((Subtarget->isNeonAvailable())) {
14709 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv8b_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
14710 }
14711 return Register();
14712}
14713
14714Register fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(uint64_t imm0) {
14715 if ((Subtarget->isNeonAvailable())) {
14716 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv16b_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14717 }
14718 return Register();
14719}
14720
14721Register fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
14722switch (RetVT.SimpleTy) {
14723 case MVT::v8i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(imm0);
14724 case MVT::v16i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(imm0);
14725 default: return Register();
14726}
14727}
14728
14729Register fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
14730 switch (VT.SimpleTy) {
14731 case MVT::i32: return fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
14732 default: return Register();
14733 }
14734}
14735
14736// FastEmit functions for AArch64ISD::MOVIedit.
14737
14738Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(uint64_t imm0) {
14739 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVID, RC: &AArch64::FPR64RegClass, Imm: imm0);
14740}
14741
14742Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(uint64_t imm0) {
14743 if ((Subtarget->isNeonAvailable())) {
14744 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv2d_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14745 }
14746 return Register();
14747}
14748
14749Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
14750switch (RetVT.SimpleTy) {
14751 case MVT::f64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(imm0);
14752 case MVT::v2i64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(imm0);
14753 default: return Register();
14754}
14755}
14756
14757Register fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
14758 switch (VT.SimpleTy) {
14759 case MVT::i32: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
14760 default: return Register();
14761 }
14762}
14763
14764// Top-level FastEmit function.
14765
14766Register fastEmit_i_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
14767 switch (Opcode) {
14768 case AArch64ISD::FMOV: return fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(VT, RetVT, imm0);
14769 case AArch64ISD::MOVI: return fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(VT, RetVT, imm0);
14770 case AArch64ISD::MOVIedit: return fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(VT, RetVT, imm0);
14771 default: return Register();
14772 }
14773}
14774
14775// FastEmit functions for AArch64ISD::RDSVL.
14776
14777Register fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(MVT RetVT, uint64_t imm0) {
14778 if (RetVT.SimpleTy != MVT::i64)
14779 return Register();
14780 if ((Subtarget->hasSME())) {
14781 return fastEmitInst_i(MachineInstOpcode: AArch64::RDSVLI_XI, RC: &AArch64::GPR64RegClass, Imm: imm0);
14782 }
14783 return Register();
14784}
14785
14786Register fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(MVT VT, MVT RetVT, uint64_t imm0) {
14787 switch (VT.SimpleTy) {
14788 case MVT::i32: return fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(RetVT, imm0);
14789 default: return Register();
14790 }
14791}
14792
14793// Top-level FastEmit function.
14794
14795Register fastEmit_i_Predicate_simm6_32b(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
14796 switch (Opcode) {
14797 case AArch64ISD::RDSVL: return fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(VT, RetVT, imm0);
14798 default: return Register();
14799 }
14800}
14801
14802