1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the ARM target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_mod_imm(int64_t Imm) {
12
13 return ARM_AM::getSOImmVal(Arg: Imm) != -1;
14
15}
16static bool Predicate_imm0_7(int64_t Imm) {
17
18 return Imm >= 0 && Imm < 8;
19
20}
21static bool Predicate_imm0_255_expr(int64_t Imm) {
22 return Imm >= 0 && Imm < 256;
23}
24static bool Predicate_imm_sr(int64_t Imm) {
25
26 return Imm > 0 && Imm <= 32;
27
28}
29static bool Predicate_t2_so_imm(int64_t Imm) {
30
31 return ARM_AM::getT2SOImmVal(Arg: Imm) != -1;
32
33}
34static bool Predicate_imm0_4095(int64_t Imm) {
35
36 return Imm >= 0 && Imm < 4096;
37
38}
39static bool Predicate_imm1_31(int64_t Imm) {
40 return Imm > 0 && Imm < 32;
41}
42static bool Predicate_shr_imm8(int64_t Imm) {
43 return Imm > 0 && Imm <= 8;
44}
45static bool Predicate_shr_imm16(int64_t Imm) {
46 return Imm > 0 && Imm <= 16;
47}
48static bool Predicate_shr_imm32(int64_t Imm) {
49 return Imm > 0 && Imm <= 32;
50}
51static bool Predicate_VectorIndex32(int64_t Imm) {
52
53 return ((uint64_t)Imm) < 2;
54
55}
56static bool Predicate_imm0_31(int64_t Imm) {
57
58 return Imm >= 0 && Imm < 32;
59
60}
61static bool Predicate_mod_imm_not(int64_t Imm) {
62
63 return ARM_AM::getSOImmVal(Arg: ~(uint32_t)Imm) != -1;
64
65}
66static bool Predicate_imm0_65535(int64_t Imm) {
67
68 return Imm >= 0 && Imm < 65536;
69
70}
71static bool Predicate_t2_so_imm_neg(int64_t Imm) {
72
73 return Imm && ARM_AM::getT2SOImmVal(Arg: -(uint32_t)Imm) != -1;
74
75}
76static bool Predicate_imm0_15(int64_t Imm) {
77
78 return Imm >= 0 && Imm < 16;
79
80}
81
82
83// FastEmit functions for ISD::GET_FPENV.
84
85Register fastEmit_ISD_GET_FPENV_MVT_i32_(MVT RetVT) {
86 if (RetVT.SimpleTy != MVT::i32)
87 return Register();
88 return fastEmitInst_(MachineInstOpcode: ARM::VMRS, RC: &ARM::GPRnopcRegClass);
89}
90
91Register fastEmit_ISD_GET_FPENV_(MVT VT, MVT RetVT) {
92 switch (VT.SimpleTy) {
93 case MVT::i32: return fastEmit_ISD_GET_FPENV_MVT_i32_(RetVT);
94 default: return Register();
95 }
96}
97
98// FastEmit functions for ISD::GET_FPMODE.
99
100Register fastEmit_ISD_GET_FPMODE_MVT_i32_(MVT RetVT) {
101 if (RetVT.SimpleTy != MVT::i32)
102 return Register();
103 return fastEmitInst_(MachineInstOpcode: ARM::VMRS, RC: &ARM::GPRnopcRegClass);
104}
105
106Register fastEmit_ISD_GET_FPMODE_(MVT VT, MVT RetVT) {
107 switch (VT.SimpleTy) {
108 case MVT::i32: return fastEmit_ISD_GET_FPMODE_MVT_i32_(RetVT);
109 default: return Register();
110 }
111}
112
113// Top-level FastEmit function.
114
115Register fastEmit_(MVT VT, MVT RetVT, unsigned Opcode) override {
116 switch (Opcode) {
117 case ISD::GET_FPENV: return fastEmit_ISD_GET_FPENV_(VT, RetVT);
118 case ISD::GET_FPMODE: return fastEmit_ISD_GET_FPMODE_(VT, RetVT);
119 default: return Register();
120 }
121}
122
123// FastEmit functions for ARMISD::VADDVs.
124
125Register fastEmit_ARMISD_VADDVs_MVT_v16i8_r(MVT RetVT, Register Op0) {
126 if (RetVT.SimpleTy != MVT::i32)
127 return Register();
128 if ((Subtarget->hasMVEIntegerOps())) {
129 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVs8no_acc, RC: &ARM::tGPREvenRegClass, Op0);
130 }
131 return Register();
132}
133
134Register fastEmit_ARMISD_VADDVs_MVT_v8i16_r(MVT RetVT, Register Op0) {
135 if (RetVT.SimpleTy != MVT::i32)
136 return Register();
137 if ((Subtarget->hasMVEIntegerOps())) {
138 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVs16no_acc, RC: &ARM::tGPREvenRegClass, Op0);
139 }
140 return Register();
141}
142
143Register fastEmit_ARMISD_VADDVs_MVT_v4i32_r(MVT RetVT, Register Op0) {
144 if (RetVT.SimpleTy != MVT::i32)
145 return Register();
146 if ((Subtarget->hasMVEIntegerOps())) {
147 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVs32no_acc, RC: &ARM::tGPREvenRegClass, Op0);
148 }
149 return Register();
150}
151
152Register fastEmit_ARMISD_VADDVs_r(MVT VT, MVT RetVT, Register Op0) {
153 switch (VT.SimpleTy) {
154 case MVT::v16i8: return fastEmit_ARMISD_VADDVs_MVT_v16i8_r(RetVT, Op0);
155 case MVT::v8i16: return fastEmit_ARMISD_VADDVs_MVT_v8i16_r(RetVT, Op0);
156 case MVT::v4i32: return fastEmit_ARMISD_VADDVs_MVT_v4i32_r(RetVT, Op0);
157 default: return Register();
158 }
159}
160
161// FastEmit functions for ARMISD::VADDVu.
162
163Register fastEmit_ARMISD_VADDVu_MVT_v16i8_r(MVT RetVT, Register Op0) {
164 if (RetVT.SimpleTy != MVT::i32)
165 return Register();
166 if ((Subtarget->hasMVEIntegerOps())) {
167 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu8no_acc, RC: &ARM::tGPREvenRegClass, Op0);
168 }
169 return Register();
170}
171
172Register fastEmit_ARMISD_VADDVu_MVT_v8i16_r(MVT RetVT, Register Op0) {
173 if (RetVT.SimpleTy != MVT::i32)
174 return Register();
175 if ((Subtarget->hasMVEIntegerOps())) {
176 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu16no_acc, RC: &ARM::tGPREvenRegClass, Op0);
177 }
178 return Register();
179}
180
181Register fastEmit_ARMISD_VADDVu_MVT_v4i32_r(MVT RetVT, Register Op0) {
182 if (RetVT.SimpleTy != MVT::i32)
183 return Register();
184 if ((Subtarget->hasMVEIntegerOps())) {
185 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu32no_acc, RC: &ARM::tGPREvenRegClass, Op0);
186 }
187 return Register();
188}
189
190Register fastEmit_ARMISD_VADDVu_r(MVT VT, MVT RetVT, Register Op0) {
191 switch (VT.SimpleTy) {
192 case MVT::v16i8: return fastEmit_ARMISD_VADDVu_MVT_v16i8_r(RetVT, Op0);
193 case MVT::v8i16: return fastEmit_ARMISD_VADDVu_MVT_v8i16_r(RetVT, Op0);
194 case MVT::v4i32: return fastEmit_ARMISD_VADDVu_MVT_v4i32_r(RetVT, Op0);
195 default: return Register();
196 }
197}
198
199// FastEmit functions for ARMISD::VDUP.
200
201Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i8_r(Register Op0) {
202 if ((Subtarget->hasNEON())) {
203 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP8d, RC: &ARM::DPRRegClass, Op0);
204 }
205 return Register();
206}
207
208Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v16i8_r(Register Op0) {
209 if ((Subtarget->hasMVEIntegerOps())) {
210 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP8, RC: &ARM::MQPRRegClass, Op0);
211 }
212 if ((Subtarget->hasNEON())) {
213 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP8q, RC: &ARM::QPRRegClass, Op0);
214 }
215 return Register();
216}
217
218Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i16_r(Register Op0) {
219 if ((Subtarget->hasNEON())) {
220 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP16d, RC: &ARM::DPRRegClass, Op0);
221 }
222 return Register();
223}
224
225Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i16_r(Register Op0) {
226 if ((Subtarget->hasMVEIntegerOps())) {
227 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP16, RC: &ARM::MQPRRegClass, Op0);
228 }
229 if ((Subtarget->hasNEON())) {
230 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP16q, RC: &ARM::QPRRegClass, Op0);
231 }
232 return Register();
233}
234
235Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v2i32_r(Register Op0) {
236 if ((!Subtarget->hasSlowVDUP32()) && (Subtarget->hasNEON())) {
237 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP32d, RC: &ARM::DPRRegClass, Op0);
238 }
239 return Register();
240}
241
242Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i32_r(Register Op0) {
243 if ((Subtarget->hasMVEIntegerOps())) {
244 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP32, RC: &ARM::MQPRRegClass, Op0);
245 }
246 if ((Subtarget->hasNEON())) {
247 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP32q, RC: &ARM::QPRRegClass, Op0);
248 }
249 return Register();
250}
251
252Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8f16_r(Register Op0) {
253 if ((Subtarget->hasMVEIntegerOps())) {
254 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP16, RC: &ARM::MQPRRegClass, Op0);
255 }
256 return Register();
257}
258
259Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4f32_r(Register Op0) {
260 if ((Subtarget->hasMVEIntegerOps())) {
261 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP32, RC: &ARM::MQPRRegClass, Op0);
262 }
263 return Register();
264}
265
266Register fastEmit_ARMISD_VDUP_MVT_i32_r(MVT RetVT, Register Op0) {
267switch (RetVT.SimpleTy) {
268 case MVT::v8i8: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i8_r(Op0);
269 case MVT::v16i8: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v16i8_r(Op0);
270 case MVT::v4i16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i16_r(Op0);
271 case MVT::v8i16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i16_r(Op0);
272 case MVT::v2i32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v2i32_r(Op0);
273 case MVT::v4i32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i32_r(Op0);
274 case MVT::v8f16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8f16_r(Op0);
275 case MVT::v4f32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4f32_r(Op0);
276 default: return Register();
277}
278}
279
280Register fastEmit_ARMISD_VDUP_r(MVT VT, MVT RetVT, Register Op0) {
281 switch (VT.SimpleTy) {
282 case MVT::i32: return fastEmit_ARMISD_VDUP_MVT_i32_r(RetVT, Op0);
283 default: return Register();
284 }
285}
286
287// FastEmit functions for ARMISD::VMOVhr.
288
289Register fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_bf16_r(Register Op0) {
290 if ((Subtarget->hasFPRegs16())) {
291 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVHR, RC: &ARM::HPRRegClass, Op0);
292 }
293 return Register();
294}
295
296Register fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_f16_r(Register Op0) {
297 if ((Subtarget->hasFPRegs16())) {
298 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVHR, RC: &ARM::HPRRegClass, Op0);
299 }
300 return Register();
301}
302
303Register fastEmit_ARMISD_VMOVhr_MVT_i32_r(MVT RetVT, Register Op0) {
304switch (RetVT.SimpleTy) {
305 case MVT::bf16: return fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_bf16_r(Op0);
306 case MVT::f16: return fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_f16_r(Op0);
307 default: return Register();
308}
309}
310
311Register fastEmit_ARMISD_VMOVhr_r(MVT VT, MVT RetVT, Register Op0) {
312 switch (VT.SimpleTy) {
313 case MVT::i32: return fastEmit_ARMISD_VMOVhr_MVT_i32_r(RetVT, Op0);
314 default: return Register();
315 }
316}
317
318// FastEmit functions for ARMISD::VMOVrh.
319
320Register fastEmit_ARMISD_VMOVrh_MVT_bf16_r(MVT RetVT, Register Op0) {
321 if (RetVT.SimpleTy != MVT::i32)
322 return Register();
323 if ((Subtarget->hasFPRegs16())) {
324 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVRH, RC: &ARM::rGPRRegClass, Op0);
325 }
326 return Register();
327}
328
329Register fastEmit_ARMISD_VMOVrh_MVT_f16_r(MVT RetVT, Register Op0) {
330 if (RetVT.SimpleTy != MVT::i32)
331 return Register();
332 if ((Subtarget->hasFPRegs16())) {
333 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVRH, RC: &ARM::rGPRRegClass, Op0);
334 }
335 return Register();
336}
337
338Register fastEmit_ARMISD_VMOVrh_r(MVT VT, MVT RetVT, Register Op0) {
339 switch (VT.SimpleTy) {
340 case MVT::bf16: return fastEmit_ARMISD_VMOVrh_MVT_bf16_r(RetVT, Op0);
341 case MVT::f16: return fastEmit_ARMISD_VMOVrh_MVT_f16_r(RetVT, Op0);
342 default: return Register();
343 }
344}
345
346// FastEmit functions for ARMISD::VMOVsr.
347
348Register fastEmit_ARMISD_VMOVsr_MVT_i32_r(MVT RetVT, Register Op0) {
349 if (RetVT.SimpleTy != MVT::f32)
350 return Register();
351 if ((Subtarget->hasFPRegs()) && (Subtarget->preferVMOVSR() ||!Subtarget->useNEONForSinglePrecisionFP())) {
352 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVSR, RC: &ARM::SPRRegClass, Op0);
353 }
354 return Register();
355}
356
357Register fastEmit_ARMISD_VMOVsr_r(MVT VT, MVT RetVT, Register Op0) {
358 switch (VT.SimpleTy) {
359 case MVT::i32: return fastEmit_ARMISD_VMOVsr_MVT_i32_r(RetVT, Op0);
360 default: return Register();
361 }
362}
363
364// FastEmit functions for ARMISD::VREV16.
365
366Register fastEmit_ARMISD_VREV16_MVT_v8i8_r(MVT RetVT, Register Op0) {
367 if (RetVT.SimpleTy != MVT::v8i8)
368 return Register();
369 if ((Subtarget->hasNEON())) {
370 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
371 }
372 return Register();
373}
374
375Register fastEmit_ARMISD_VREV16_MVT_v16i8_r(MVT RetVT, Register Op0) {
376 if (RetVT.SimpleTy != MVT::v16i8)
377 return Register();
378 if ((Subtarget->hasMVEIntegerOps())) {
379 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
380 }
381 if ((Subtarget->hasNEON())) {
382 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
383 }
384 return Register();
385}
386
387Register fastEmit_ARMISD_VREV16_r(MVT VT, MVT RetVT, Register Op0) {
388 switch (VT.SimpleTy) {
389 case MVT::v8i8: return fastEmit_ARMISD_VREV16_MVT_v8i8_r(RetVT, Op0);
390 case MVT::v16i8: return fastEmit_ARMISD_VREV16_MVT_v16i8_r(RetVT, Op0);
391 default: return Register();
392 }
393}
394
395// FastEmit functions for ARMISD::VREV32.
396
397Register fastEmit_ARMISD_VREV32_MVT_v8i8_r(MVT RetVT, Register Op0) {
398 if (RetVT.SimpleTy != MVT::v8i8)
399 return Register();
400 if ((Subtarget->hasNEON())) {
401 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
402 }
403 return Register();
404}
405
406Register fastEmit_ARMISD_VREV32_MVT_v16i8_r(MVT RetVT, Register Op0) {
407 if (RetVT.SimpleTy != MVT::v16i8)
408 return Register();
409 if ((Subtarget->hasMVEIntegerOps())) {
410 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
411 }
412 if ((Subtarget->hasNEON())) {
413 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
414 }
415 return Register();
416}
417
418Register fastEmit_ARMISD_VREV32_MVT_v4i16_r(MVT RetVT, Register Op0) {
419 if (RetVT.SimpleTy != MVT::v4i16)
420 return Register();
421 if ((Subtarget->hasNEON())) {
422 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
423 }
424 return Register();
425}
426
427Register fastEmit_ARMISD_VREV32_MVT_v8i16_r(MVT RetVT, Register Op0) {
428 if (RetVT.SimpleTy != MVT::v8i16)
429 return Register();
430 if ((Subtarget->hasMVEIntegerOps())) {
431 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
432 }
433 if ((Subtarget->hasNEON())) {
434 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
435 }
436 return Register();
437}
438
439Register fastEmit_ARMISD_VREV32_MVT_v4f16_r(MVT RetVT, Register Op0) {
440 if (RetVT.SimpleTy != MVT::v4f16)
441 return Register();
442 if ((Subtarget->hasNEON())) {
443 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
444 }
445 return Register();
446}
447
448Register fastEmit_ARMISD_VREV32_MVT_v8f16_r(MVT RetVT, Register Op0) {
449 if (RetVT.SimpleTy != MVT::v8f16)
450 return Register();
451 if ((Subtarget->hasMVEIntegerOps())) {
452 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
453 }
454 if ((Subtarget->hasNEON())) {
455 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
456 }
457 return Register();
458}
459
460Register fastEmit_ARMISD_VREV32_MVT_v4bf16_r(MVT RetVT, Register Op0) {
461 if (RetVT.SimpleTy != MVT::v4bf16)
462 return Register();
463 if ((Subtarget->hasNEON())) {
464 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
465 }
466 return Register();
467}
468
469Register fastEmit_ARMISD_VREV32_MVT_v8bf16_r(MVT RetVT, Register Op0) {
470 if (RetVT.SimpleTy != MVT::v8bf16)
471 return Register();
472 if ((Subtarget->hasNEON())) {
473 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
474 }
475 return Register();
476}
477
478Register fastEmit_ARMISD_VREV32_r(MVT VT, MVT RetVT, Register Op0) {
479 switch (VT.SimpleTy) {
480 case MVT::v8i8: return fastEmit_ARMISD_VREV32_MVT_v8i8_r(RetVT, Op0);
481 case MVT::v16i8: return fastEmit_ARMISD_VREV32_MVT_v16i8_r(RetVT, Op0);
482 case MVT::v4i16: return fastEmit_ARMISD_VREV32_MVT_v4i16_r(RetVT, Op0);
483 case MVT::v8i16: return fastEmit_ARMISD_VREV32_MVT_v8i16_r(RetVT, Op0);
484 case MVT::v4f16: return fastEmit_ARMISD_VREV32_MVT_v4f16_r(RetVT, Op0);
485 case MVT::v8f16: return fastEmit_ARMISD_VREV32_MVT_v8f16_r(RetVT, Op0);
486 case MVT::v4bf16: return fastEmit_ARMISD_VREV32_MVT_v4bf16_r(RetVT, Op0);
487 case MVT::v8bf16: return fastEmit_ARMISD_VREV32_MVT_v8bf16_r(RetVT, Op0);
488 default: return Register();
489 }
490}
491
492// FastEmit functions for ARMISD::VREV64.
493
494Register fastEmit_ARMISD_VREV64_MVT_v8i8_r(MVT RetVT, Register Op0) {
495 if (RetVT.SimpleTy != MVT::v8i8)
496 return Register();
497 if ((Subtarget->hasNEON())) {
498 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
499 }
500 return Register();
501}
502
503Register fastEmit_ARMISD_VREV64_MVT_v16i8_r(MVT RetVT, Register Op0) {
504 if (RetVT.SimpleTy != MVT::v16i8)
505 return Register();
506 if ((Subtarget->hasMVEIntegerOps())) {
507 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
508 }
509 if ((Subtarget->hasNEON())) {
510 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
511 }
512 return Register();
513}
514
515Register fastEmit_ARMISD_VREV64_MVT_v4i16_r(MVT RetVT, Register Op0) {
516 if (RetVT.SimpleTy != MVT::v4i16)
517 return Register();
518 if ((Subtarget->hasNEON())) {
519 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
520 }
521 return Register();
522}
523
524Register fastEmit_ARMISD_VREV64_MVT_v8i16_r(MVT RetVT, Register Op0) {
525 if (RetVT.SimpleTy != MVT::v8i16)
526 return Register();
527 if ((Subtarget->hasMVEIntegerOps())) {
528 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
529 }
530 if ((Subtarget->hasNEON())) {
531 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
532 }
533 return Register();
534}
535
536Register fastEmit_ARMISD_VREV64_MVT_v2i32_r(MVT RetVT, Register Op0) {
537 if (RetVT.SimpleTy != MVT::v2i32)
538 return Register();
539 if ((Subtarget->hasNEON())) {
540 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
541 }
542 return Register();
543}
544
545Register fastEmit_ARMISD_VREV64_MVT_v4i32_r(MVT RetVT, Register Op0) {
546 if (RetVT.SimpleTy != MVT::v4i32)
547 return Register();
548 if ((Subtarget->hasMVEIntegerOps())) {
549 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
550 }
551 if ((Subtarget->hasNEON())) {
552 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
553 }
554 return Register();
555}
556
557Register fastEmit_ARMISD_VREV64_MVT_v4f16_r(MVT RetVT, Register Op0) {
558 if (RetVT.SimpleTy != MVT::v4f16)
559 return Register();
560 if ((Subtarget->hasNEON())) {
561 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
562 }
563 return Register();
564}
565
566Register fastEmit_ARMISD_VREV64_MVT_v8f16_r(MVT RetVT, Register Op0) {
567 if (RetVT.SimpleTy != MVT::v8f16)
568 return Register();
569 if ((Subtarget->hasMVEIntegerOps())) {
570 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
571 }
572 if ((Subtarget->hasNEON())) {
573 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
574 }
575 return Register();
576}
577
578Register fastEmit_ARMISD_VREV64_MVT_v4bf16_r(MVT RetVT, Register Op0) {
579 if (RetVT.SimpleTy != MVT::v4bf16)
580 return Register();
581 if ((Subtarget->hasNEON())) {
582 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
583 }
584 return Register();
585}
586
587Register fastEmit_ARMISD_VREV64_MVT_v8bf16_r(MVT RetVT, Register Op0) {
588 if (RetVT.SimpleTy != MVT::v8bf16)
589 return Register();
590 if ((Subtarget->hasNEON())) {
591 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
592 }
593 return Register();
594}
595
596Register fastEmit_ARMISD_VREV64_MVT_v2f32_r(MVT RetVT, Register Op0) {
597 if (RetVT.SimpleTy != MVT::v2f32)
598 return Register();
599 if ((Subtarget->hasNEON())) {
600 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
601 }
602 return Register();
603}
604
605Register fastEmit_ARMISD_VREV64_MVT_v4f32_r(MVT RetVT, Register Op0) {
606 if (RetVT.SimpleTy != MVT::v4f32)
607 return Register();
608 if ((Subtarget->hasMVEIntegerOps())) {
609 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
610 }
611 if ((Subtarget->hasNEON())) {
612 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
613 }
614 return Register();
615}
616
617Register fastEmit_ARMISD_VREV64_r(MVT VT, MVT RetVT, Register Op0) {
618 switch (VT.SimpleTy) {
619 case MVT::v8i8: return fastEmit_ARMISD_VREV64_MVT_v8i8_r(RetVT, Op0);
620 case MVT::v16i8: return fastEmit_ARMISD_VREV64_MVT_v16i8_r(RetVT, Op0);
621 case MVT::v4i16: return fastEmit_ARMISD_VREV64_MVT_v4i16_r(RetVT, Op0);
622 case MVT::v8i16: return fastEmit_ARMISD_VREV64_MVT_v8i16_r(RetVT, Op0);
623 case MVT::v2i32: return fastEmit_ARMISD_VREV64_MVT_v2i32_r(RetVT, Op0);
624 case MVT::v4i32: return fastEmit_ARMISD_VREV64_MVT_v4i32_r(RetVT, Op0);
625 case MVT::v4f16: return fastEmit_ARMISD_VREV64_MVT_v4f16_r(RetVT, Op0);
626 case MVT::v8f16: return fastEmit_ARMISD_VREV64_MVT_v8f16_r(RetVT, Op0);
627 case MVT::v4bf16: return fastEmit_ARMISD_VREV64_MVT_v4bf16_r(RetVT, Op0);
628 case MVT::v8bf16: return fastEmit_ARMISD_VREV64_MVT_v8bf16_r(RetVT, Op0);
629 case MVT::v2f32: return fastEmit_ARMISD_VREV64_MVT_v2f32_r(RetVT, Op0);
630 case MVT::v4f32: return fastEmit_ARMISD_VREV64_MVT_v4f32_r(RetVT, Op0);
631 default: return Register();
632 }
633}
634
635// FastEmit functions for ARMISD::WLSSETUP.
636
637Register fastEmit_ARMISD_WLSSETUP_MVT_i32_r(MVT RetVT, Register Op0) {
638 if (RetVT.SimpleTy != MVT::i32)
639 return Register();
640 if ((Subtarget->hasLOB()) && (Subtarget->hasV8_1MMainlineOps()) && (Subtarget->isThumb2())) {
641 return fastEmitInst_r(MachineInstOpcode: ARM::t2WhileLoopSetup, RC: &ARM::GPRlrRegClass, Op0);
642 }
643 return Register();
644}
645
646Register fastEmit_ARMISD_WLSSETUP_r(MVT VT, MVT RetVT, Register Op0) {
647 switch (VT.SimpleTy) {
648 case MVT::i32: return fastEmit_ARMISD_WLSSETUP_MVT_i32_r(RetVT, Op0);
649 default: return Register();
650 }
651}
652
653// FastEmit functions for ISD::ABS.
654
655Register fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, Register Op0) {
656 if (RetVT.SimpleTy != MVT::v8i8)
657 return Register();
658 if ((Subtarget->hasNEON())) {
659 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv8i8, RC: &ARM::DPRRegClass, Op0);
660 }
661 return Register();
662}
663
664Register fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, Register Op0) {
665 if (RetVT.SimpleTy != MVT::v16i8)
666 return Register();
667 if ((Subtarget->hasMVEIntegerOps())) {
668 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSs8, RC: &ARM::MQPRRegClass, Op0);
669 }
670 if ((Subtarget->hasNEON())) {
671 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv16i8, RC: &ARM::QPRRegClass, Op0);
672 }
673 return Register();
674}
675
676Register fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, Register Op0) {
677 if (RetVT.SimpleTy != MVT::v4i16)
678 return Register();
679 if ((Subtarget->hasNEON())) {
680 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv4i16, RC: &ARM::DPRRegClass, Op0);
681 }
682 return Register();
683}
684
685Register fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, Register Op0) {
686 if (RetVT.SimpleTy != MVT::v8i16)
687 return Register();
688 if ((Subtarget->hasMVEIntegerOps())) {
689 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSs16, RC: &ARM::MQPRRegClass, Op0);
690 }
691 if ((Subtarget->hasNEON())) {
692 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv8i16, RC: &ARM::QPRRegClass, Op0);
693 }
694 return Register();
695}
696
697Register fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, Register Op0) {
698 if (RetVT.SimpleTy != MVT::v2i32)
699 return Register();
700 if ((Subtarget->hasNEON())) {
701 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv2i32, RC: &ARM::DPRRegClass, Op0);
702 }
703 return Register();
704}
705
706Register fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, Register Op0) {
707 if (RetVT.SimpleTy != MVT::v4i32)
708 return Register();
709 if ((Subtarget->hasMVEIntegerOps())) {
710 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSs32, RC: &ARM::MQPRRegClass, Op0);
711 }
712 if ((Subtarget->hasNEON())) {
713 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv4i32, RC: &ARM::QPRRegClass, Op0);
714 }
715 return Register();
716}
717
718Register fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, Register Op0) {
719 switch (VT.SimpleTy) {
720 case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0);
721 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
722 case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0);
723 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
724 case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0);
725 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
726 default: return Register();
727 }
728}
729
730// FastEmit functions for ISD::ANY_EXTEND.
731
732Register fastEmit_ISD_ANY_EXTEND_MVT_v8i8_r(MVT RetVT, Register Op0) {
733 if (RetVT.SimpleTy != MVT::v8i16)
734 return Register();
735 if ((Subtarget->hasNEON())) {
736 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv8i16, RC: &ARM::QPRRegClass, Op0);
737 }
738 return Register();
739}
740
741Register fastEmit_ISD_ANY_EXTEND_MVT_v4i16_r(MVT RetVT, Register Op0) {
742 if (RetVT.SimpleTy != MVT::v4i32)
743 return Register();
744 if ((Subtarget->hasNEON())) {
745 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv4i32, RC: &ARM::QPRRegClass, Op0);
746 }
747 return Register();
748}
749
750Register fastEmit_ISD_ANY_EXTEND_MVT_v2i32_r(MVT RetVT, Register Op0) {
751 if (RetVT.SimpleTy != MVT::v2i64)
752 return Register();
753 if ((Subtarget->hasNEON())) {
754 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv2i64, RC: &ARM::QPRRegClass, Op0);
755 }
756 return Register();
757}
758
759Register fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
760 switch (VT.SimpleTy) {
761 case MVT::v8i8: return fastEmit_ISD_ANY_EXTEND_MVT_v8i8_r(RetVT, Op0);
762 case MVT::v4i16: return fastEmit_ISD_ANY_EXTEND_MVT_v4i16_r(RetVT, Op0);
763 case MVT::v2i32: return fastEmit_ISD_ANY_EXTEND_MVT_v2i32_r(RetVT, Op0);
764 default: return Register();
765 }
766}
767
768// FastEmit functions for ISD::BITCAST.
769
770Register fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, Register Op0) {
771 if (RetVT.SimpleTy != MVT::f32)
772 return Register();
773 if ((Subtarget->hasFPRegs()) && (Subtarget->preferVMOVSR() ||!Subtarget->useNEONForSinglePrecisionFP())) {
774 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVSR, RC: &ARM::SPRRegClass, Op0);
775 }
776 return Register();
777}
778
779Register fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, Register Op0) {
780 if (RetVT.SimpleTy != MVT::i32)
781 return Register();
782 if ((Subtarget->hasFPRegs())) {
783 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVRS, RC: &ARM::GPRRegClass, Op0);
784 }
785 return Register();
786}
787
788Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Register Op0) {
789 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
790 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
791 }
792 return Register();
793}
794
795Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Register Op0) {
796 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
797 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
798 }
799 return Register();
800}
801
802Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Register Op0) {
803 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
804 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
805 }
806 return Register();
807}
808
809Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Register Op0) {
810 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
811 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
812 }
813 return Register();
814}
815
816Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Register Op0) {
817 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
818 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
819 }
820 return Register();
821}
822
823Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Register Op0) {
824 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
825 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
826 }
827 return Register();
828}
829
830Register fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, Register Op0) {
831switch (RetVT.SimpleTy) {
832 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0);
833 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0);
834 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0);
835 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0);
836 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Op0);
837 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0);
838 default: return Register();
839}
840}
841
842Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Register Op0) {
843 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
844 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
845 }
846 return Register();
847}
848
849Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Register Op0) {
850 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
851 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
852 }
853 return Register();
854}
855
856Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Register Op0) {
857 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
858 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
859 }
860 return Register();
861}
862
863Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Register Op0) {
864 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
865 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
866 }
867 return Register();
868}
869
870Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Register Op0) {
871 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
872 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
873 }
874 return Register();
875}
876
877Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Register Op0) {
878 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
879 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
880 }
881 return Register();
882}
883
884Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Register Op0) {
885 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
886 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
887 }
888 return Register();
889}
890
891Register fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, Register Op0) {
892switch (RetVT.SimpleTy) {
893 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0);
894 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0);
895 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0);
896 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0);
897 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0);
898 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Op0);
899 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0);
900 default: return Register();
901}
902}
903
904Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Register Op0) {
905 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
906 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
907 }
908 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
909 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
910 }
911 return Register();
912}
913
914Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Register Op0) {
915 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
916 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
917 }
918 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
919 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
920 }
921 return Register();
922}
923
924Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Register Op0) {
925 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
926 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
927 }
928 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
929 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
930 }
931 return Register();
932}
933
934Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Register Op0) {
935 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
936 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
937 }
938 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
939 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
940 }
941 return Register();
942}
943
944Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Register Op0) {
945 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
946 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
947 }
948 return Register();
949}
950
951Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Register Op0) {
952 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
953 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
954 }
955 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
956 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
957 }
958 return Register();
959}
960
961Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Register Op0) {
962 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
963 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
964 }
965 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
966 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
967 }
968 return Register();
969}
970
971Register fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, Register Op0) {
972switch (RetVT.SimpleTy) {
973 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0);
974 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0);
975 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0);
976 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0);
977 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Op0);
978 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0);
979 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0);
980 default: return Register();
981}
982}
983
984Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Register Op0) {
985 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
986 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
987 }
988 return Register();
989}
990
991Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Register Op0) {
992 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
993 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
994 }
995 return Register();
996}
997
998Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Register Op0) {
999 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1000 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1001 }
1002 return Register();
1003}
1004
1005Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Register Op0) {
1006 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1007 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1008 }
1009 return Register();
1010}
1011
1012Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Register Op0) {
1013 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1014 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1015 }
1016 return Register();
1017}
1018
1019Register fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, Register Op0) {
1020switch (RetVT.SimpleTy) {
1021 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0);
1022 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0);
1023 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0);
1024 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0);
1025 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0);
1026 default: return Register();
1027}
1028}
1029
1030Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Register Op0) {
1031 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1032 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1033 }
1034 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1035 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1036 }
1037 return Register();
1038}
1039
1040Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Register Op0) {
1041 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1042 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1043 }
1044 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1045 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1046 }
1047 return Register();
1048}
1049
1050Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Register Op0) {
1051 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1052 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1053 }
1054 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1055 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1056 }
1057 return Register();
1058}
1059
1060Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Register Op0) {
1061 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1062 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1063 }
1064 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1065 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1066 }
1067 return Register();
1068}
1069
1070Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Register Op0) {
1071 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1072 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1073 }
1074 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1075 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1076 }
1077 return Register();
1078}
1079
1080Register fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, Register Op0) {
1081switch (RetVT.SimpleTy) {
1082 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0);
1083 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0);
1084 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0);
1085 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0);
1086 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0);
1087 default: return Register();
1088}
1089}
1090
1091Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Register Op0) {
1092 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1093 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1094 }
1095 return Register();
1096}
1097
1098Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Register Op0) {
1099 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1100 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1101 }
1102 return Register();
1103}
1104
1105Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Register Op0) {
1106 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1107 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1108 }
1109 return Register();
1110}
1111
1112Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Register Op0) {
1113 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1114 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1115 }
1116 return Register();
1117}
1118
1119Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Register Op0) {
1120 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1121 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1122 }
1123 return Register();
1124}
1125
1126Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Register Op0) {
1127 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1128 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1129 }
1130 return Register();
1131}
1132
1133Register fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, Register Op0) {
1134switch (RetVT.SimpleTy) {
1135 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0);
1136 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0);
1137 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0);
1138 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0);
1139 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0);
1140 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Op0);
1141 default: return Register();
1142}
1143}
1144
1145Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Register Op0) {
1146 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1147 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1148 }
1149 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1150 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1151 }
1152 return Register();
1153}
1154
1155Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Register Op0) {
1156 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1157 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1158 }
1159 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1160 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1161 }
1162 return Register();
1163}
1164
1165Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Register Op0) {
1166 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1167 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1168 }
1169 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1170 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1171 }
1172 return Register();
1173}
1174
1175Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Register Op0) {
1176 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1177 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1178 }
1179 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1180 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1181 }
1182 return Register();
1183}
1184
1185Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Register Op0) {
1186 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1187 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1188 }
1189 return Register();
1190}
1191
1192Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Register Op0) {
1193 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1194 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1195 }
1196 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1197 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1198 }
1199 return Register();
1200}
1201
1202Register fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, Register Op0) {
1203switch (RetVT.SimpleTy) {
1204 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0);
1205 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0);
1206 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0);
1207 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0);
1208 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Op0);
1209 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0);
1210 default: return Register();
1211}
1212}
1213
1214Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Register Op0) {
1215 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1216 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
1217 }
1218 return Register();
1219}
1220
1221Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Register Op0) {
1222 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1223 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1224 }
1225 return Register();
1226}
1227
1228Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Register Op0) {
1229 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1230 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1231 }
1232 return Register();
1233}
1234
1235Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Register Op0) {
1236 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1237 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1238 }
1239 return Register();
1240}
1241
1242Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Register Op0) {
1243 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1244 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1245 }
1246 return Register();
1247}
1248
1249Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Register Op0) {
1250 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1251 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1252 }
1253 return Register();
1254}
1255
1256Register fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, Register Op0) {
1257switch (RetVT.SimpleTy) {
1258 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0);
1259 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0);
1260 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0);
1261 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0);
1262 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Op0);
1263 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0);
1264 default: return Register();
1265}
1266}
1267
1268Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Register Op0) {
1269 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1270 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1271 }
1272 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1273 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1274 }
1275 return Register();
1276}
1277
1278Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Register Op0) {
1279 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1280 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1281 }
1282 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1283 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1284 }
1285 return Register();
1286}
1287
1288Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Register Op0) {
1289 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1290 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1291 }
1292 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1293 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1294 }
1295 return Register();
1296}
1297
1298Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Register Op0) {
1299 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1300 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1301 }
1302 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1303 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1304 }
1305 return Register();
1306}
1307
1308Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Register Op0) {
1309 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1310 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1311 }
1312 return Register();
1313}
1314
1315Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Register Op0) {
1316 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1317 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1318 }
1319 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1320 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1321 }
1322 return Register();
1323}
1324
1325Register fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, Register Op0) {
1326switch (RetVT.SimpleTy) {
1327 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0);
1328 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0);
1329 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0);
1330 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0);
1331 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Op0);
1332 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0);
1333 default: return Register();
1334}
1335}
1336
1337Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Register Op0) {
1338 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1339 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1340 }
1341 return Register();
1342}
1343
1344Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Register Op0) {
1345 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1346 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1347 }
1348 return Register();
1349}
1350
1351Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Register Op0) {
1352 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1353 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1354 }
1355 return Register();
1356}
1357
1358Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Register Op0) {
1359 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1360 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1361 }
1362 return Register();
1363}
1364
1365Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Register Op0) {
1366 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1367 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1368 }
1369 return Register();
1370}
1371
1372Register fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, Register Op0) {
1373switch (RetVT.SimpleTy) {
1374 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0);
1375 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0);
1376 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0);
1377 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0);
1378 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0);
1379 default: return Register();
1380}
1381}
1382
1383Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Register Op0) {
1384 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1385 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1386 }
1387 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1388 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1389 }
1390 return Register();
1391}
1392
1393Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Register Op0) {
1394 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1395 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1396 }
1397 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1398 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1399 }
1400 return Register();
1401}
1402
1403Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Register Op0) {
1404 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1405 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1406 }
1407 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1408 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1409 }
1410 return Register();
1411}
1412
1413Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Register Op0) {
1414 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1415 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1416 }
1417 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1418 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1419 }
1420 return Register();
1421}
1422
1423Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Register Op0) {
1424 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1425 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1426 }
1427 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1428 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1429 }
1430 return Register();
1431}
1432
1433Register fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, Register Op0) {
1434switch (RetVT.SimpleTy) {
1435 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0);
1436 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0);
1437 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0);
1438 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0);
1439 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0);
1440 default: return Register();
1441}
1442}
1443
1444Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Register Op0) {
1445 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1446 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1447 }
1448 return Register();
1449}
1450
1451Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Register Op0) {
1452 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1453 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1454 }
1455 return Register();
1456}
1457
1458Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Register Op0) {
1459 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1460 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1461 }
1462 return Register();
1463}
1464
1465Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Register Op0) {
1466 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1467 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1468 }
1469 return Register();
1470}
1471
1472Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Register Op0) {
1473 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1474 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1475 }
1476 return Register();
1477}
1478
1479Register fastEmit_ISD_BITCAST_MVT_v4bf16_r(MVT RetVT, Register Op0) {
1480switch (RetVT.SimpleTy) {
1481 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Op0);
1482 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Op0);
1483 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Op0);
1484 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Op0);
1485 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Op0);
1486 default: return Register();
1487}
1488}
1489
1490Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Register Op0) {
1491 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1492 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1493 }
1494 return Register();
1495}
1496
1497Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Register Op0) {
1498 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1499 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1500 }
1501 return Register();
1502}
1503
1504Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Register Op0) {
1505 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1506 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1507 }
1508 return Register();
1509}
1510
1511Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Register Op0) {
1512 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1513 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1514 }
1515 return Register();
1516}
1517
1518Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Register Op0) {
1519 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1520 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1521 }
1522 return Register();
1523}
1524
1525Register fastEmit_ISD_BITCAST_MVT_v8bf16_r(MVT RetVT, Register Op0) {
1526switch (RetVT.SimpleTy) {
1527 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Op0);
1528 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Op0);
1529 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Op0);
1530 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Op0);
1531 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Op0);
1532 default: return Register();
1533}
1534}
1535
1536Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Register Op0) {
1537 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1538 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1539 }
1540 return Register();
1541}
1542
1543Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Register Op0) {
1544 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1545 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1546 }
1547 return Register();
1548}
1549
1550Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Register Op0) {
1551 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1552 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1553 }
1554 return Register();
1555}
1556
1557Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Register Op0) {
1558 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1559 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1560 }
1561 return Register();
1562}
1563
1564Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Register Op0) {
1565 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1566 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1567 }
1568 return Register();
1569}
1570
1571Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Register Op0) {
1572 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1573 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1574 }
1575 return Register();
1576}
1577
1578Register fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, Register Op0) {
1579switch (RetVT.SimpleTy) {
1580 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0);
1581 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0);
1582 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0);
1583 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0);
1584 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0);
1585 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Op0);
1586 default: return Register();
1587}
1588}
1589
1590Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Register Op0) {
1591 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1592 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1593 }
1594 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1595 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1596 }
1597 return Register();
1598}
1599
1600Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Register Op0) {
1601 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1602 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1603 }
1604 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1605 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1606 }
1607 return Register();
1608}
1609
1610Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Register Op0) {
1611 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1612 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1613 }
1614 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1615 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1616 }
1617 return Register();
1618}
1619
1620Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Register Op0) {
1621 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1622 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1623 }
1624 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1625 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1626 }
1627 return Register();
1628}
1629
1630Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Register Op0) {
1631 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1632 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1633 }
1634 return Register();
1635}
1636
1637Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Register Op0) {
1638 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1639 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1640 }
1641 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1642 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1643 }
1644 return Register();
1645}
1646
1647Register fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, Register Op0) {
1648switch (RetVT.SimpleTy) {
1649 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0);
1650 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0);
1651 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0);
1652 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0);
1653 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Op0);
1654 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0);
1655 default: return Register();
1656}
1657}
1658
1659Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Register Op0) {
1660 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1661 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1662 }
1663 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1664 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1665 }
1666 return Register();
1667}
1668
1669Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Register Op0) {
1670 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1671 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1672 }
1673 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1674 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1675 }
1676 return Register();
1677}
1678
1679Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Register Op0) {
1680 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1681 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1682 }
1683 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1684 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1685 }
1686 return Register();
1687}
1688
1689Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Register Op0) {
1690 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1691 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1692 }
1693 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1694 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1695 }
1696 return Register();
1697}
1698
1699Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Register Op0) {
1700 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1701 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1702 }
1703 return Register();
1704}
1705
1706Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Register Op0) {
1707 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1708 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1709 }
1710 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1711 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1712 }
1713 return Register();
1714}
1715
1716Register fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, Register Op0) {
1717switch (RetVT.SimpleTy) {
1718 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0);
1719 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0);
1720 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0);
1721 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0);
1722 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Op0);
1723 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0);
1724 default: return Register();
1725}
1726}
1727
1728Register fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, Register Op0) {
1729 switch (VT.SimpleTy) {
1730 case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0);
1731 case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0);
1732 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
1733 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0);
1734 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0);
1735 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0);
1736 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0);
1737 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0);
1738 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0);
1739 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0);
1740 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0);
1741 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0);
1742 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0);
1743 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v4bf16_r(RetVT, Op0);
1744 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v8bf16_r(RetVT, Op0);
1745 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0);
1746 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0);
1747 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0);
1748 default: return Register();
1749 }
1750}
1751
1752// FastEmit functions for ISD::BITREVERSE.
1753
1754Register fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, Register Op0) {
1755 if (RetVT.SimpleTy != MVT::i32)
1756 return Register();
1757 if ((Subtarget->isThumb2())) {
1758 return fastEmitInst_r(MachineInstOpcode: ARM::t2RBIT, RC: &ARM::rGPRRegClass, Op0);
1759 }
1760 if ((Subtarget->hasV6T2Ops()) && (!Subtarget->isThumb())) {
1761 return fastEmitInst_r(MachineInstOpcode: ARM::RBIT, RC: &ARM::GPRRegClass, Op0);
1762 }
1763 return Register();
1764}
1765
1766Register fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, Register Op0) {
1767 switch (VT.SimpleTy) {
1768 case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0);
1769 default: return Register();
1770 }
1771}
1772
1773// FastEmit functions for ISD::BSWAP.
1774
1775Register fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, Register Op0) {
1776 if (RetVT.SimpleTy != MVT::i32)
1777 return Register();
1778 if ((Subtarget->isThumb2())) {
1779 return fastEmitInst_r(MachineInstOpcode: ARM::t2REV, RC: &ARM::rGPRRegClass, Op0);
1780 }
1781 if ((Subtarget->hasV6Ops()) && (Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
1782 return fastEmitInst_r(MachineInstOpcode: ARM::tREV, RC: &ARM::tGPRRegClass, Op0);
1783 }
1784 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
1785 return fastEmitInst_r(MachineInstOpcode: ARM::REV, RC: &ARM::GPRRegClass, Op0);
1786 }
1787 return Register();
1788}
1789
1790Register fastEmit_ISD_BSWAP_MVT_v8i16_r(MVT RetVT, Register Op0) {
1791 if (RetVT.SimpleTy != MVT::v8i16)
1792 return Register();
1793 if ((Subtarget->hasMVEIntegerOps())) {
1794 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1795 }
1796 return Register();
1797}
1798
1799Register fastEmit_ISD_BSWAP_MVT_v4i32_r(MVT RetVT, Register Op0) {
1800 if (RetVT.SimpleTy != MVT::v4i32)
1801 return Register();
1802 if ((Subtarget->hasMVEIntegerOps())) {
1803 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1804 }
1805 return Register();
1806}
1807
1808Register fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, Register Op0) {
1809 switch (VT.SimpleTy) {
1810 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
1811 case MVT::v8i16: return fastEmit_ISD_BSWAP_MVT_v8i16_r(RetVT, Op0);
1812 case MVT::v4i32: return fastEmit_ISD_BSWAP_MVT_v4i32_r(RetVT, Op0);
1813 default: return Register();
1814 }
1815}
1816
1817// FastEmit functions for ISD::CTLS.
1818
1819Register fastEmit_ISD_CTLS_MVT_v8i8_r(MVT RetVT, Register Op0) {
1820 if (RetVT.SimpleTy != MVT::v8i8)
1821 return Register();
1822 if ((Subtarget->hasNEON())) {
1823 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv8i8, RC: &ARM::DPRRegClass, Op0);
1824 }
1825 return Register();
1826}
1827
1828Register fastEmit_ISD_CTLS_MVT_v16i8_r(MVT RetVT, Register Op0) {
1829 if (RetVT.SimpleTy != MVT::v16i8)
1830 return Register();
1831 if ((Subtarget->hasMVEIntegerOps())) {
1832 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLSs8, RC: &ARM::MQPRRegClass, Op0);
1833 }
1834 if ((Subtarget->hasNEON())) {
1835 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv16i8, RC: &ARM::QPRRegClass, Op0);
1836 }
1837 return Register();
1838}
1839
1840Register fastEmit_ISD_CTLS_MVT_v4i16_r(MVT RetVT, Register Op0) {
1841 if (RetVT.SimpleTy != MVT::v4i16)
1842 return Register();
1843 if ((Subtarget->hasNEON())) {
1844 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv4i16, RC: &ARM::DPRRegClass, Op0);
1845 }
1846 return Register();
1847}
1848
1849Register fastEmit_ISD_CTLS_MVT_v8i16_r(MVT RetVT, Register Op0) {
1850 if (RetVT.SimpleTy != MVT::v8i16)
1851 return Register();
1852 if ((Subtarget->hasMVEIntegerOps())) {
1853 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLSs16, RC: &ARM::MQPRRegClass, Op0);
1854 }
1855 if ((Subtarget->hasNEON())) {
1856 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv8i16, RC: &ARM::QPRRegClass, Op0);
1857 }
1858 return Register();
1859}
1860
1861Register fastEmit_ISD_CTLS_MVT_v2i32_r(MVT RetVT, Register Op0) {
1862 if (RetVT.SimpleTy != MVT::v2i32)
1863 return Register();
1864 if ((Subtarget->hasNEON())) {
1865 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv2i32, RC: &ARM::DPRRegClass, Op0);
1866 }
1867 return Register();
1868}
1869
1870Register fastEmit_ISD_CTLS_MVT_v4i32_r(MVT RetVT, Register Op0) {
1871 if (RetVT.SimpleTy != MVT::v4i32)
1872 return Register();
1873 if ((Subtarget->hasMVEIntegerOps())) {
1874 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLSs32, RC: &ARM::MQPRRegClass, Op0);
1875 }
1876 if ((Subtarget->hasNEON())) {
1877 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv4i32, RC: &ARM::QPRRegClass, Op0);
1878 }
1879 return Register();
1880}
1881
1882Register fastEmit_ISD_CTLS_r(MVT VT, MVT RetVT, Register Op0) {
1883 switch (VT.SimpleTy) {
1884 case MVT::v8i8: return fastEmit_ISD_CTLS_MVT_v8i8_r(RetVT, Op0);
1885 case MVT::v16i8: return fastEmit_ISD_CTLS_MVT_v16i8_r(RetVT, Op0);
1886 case MVT::v4i16: return fastEmit_ISD_CTLS_MVT_v4i16_r(RetVT, Op0);
1887 case MVT::v8i16: return fastEmit_ISD_CTLS_MVT_v8i16_r(RetVT, Op0);
1888 case MVT::v2i32: return fastEmit_ISD_CTLS_MVT_v2i32_r(RetVT, Op0);
1889 case MVT::v4i32: return fastEmit_ISD_CTLS_MVT_v4i32_r(RetVT, Op0);
1890 default: return Register();
1891 }
1892}
1893
1894// FastEmit functions for ISD::CTLZ.
1895
1896Register fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, Register Op0) {
1897 if (RetVT.SimpleTy != MVT::i32)
1898 return Register();
1899 if ((Subtarget->isThumb2())) {
1900 return fastEmitInst_r(MachineInstOpcode: ARM::t2CLZ, RC: &ARM::rGPRRegClass, Op0);
1901 }
1902 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb())) {
1903 return fastEmitInst_r(MachineInstOpcode: ARM::CLZ, RC: &ARM::GPRRegClass, Op0);
1904 }
1905 return Register();
1906}
1907
1908Register fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, Register Op0) {
1909 if (RetVT.SimpleTy != MVT::v8i8)
1910 return Register();
1911 if ((Subtarget->hasNEON())) {
1912 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv8i8, RC: &ARM::DPRRegClass, Op0);
1913 }
1914 return Register();
1915}
1916
1917Register fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, Register Op0) {
1918 if (RetVT.SimpleTy != MVT::v16i8)
1919 return Register();
1920 if ((Subtarget->hasMVEIntegerOps())) {
1921 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLZs8, RC: &ARM::MQPRRegClass, Op0);
1922 }
1923 if ((Subtarget->hasNEON())) {
1924 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv16i8, RC: &ARM::QPRRegClass, Op0);
1925 }
1926 return Register();
1927}
1928
1929Register fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, Register Op0) {
1930 if (RetVT.SimpleTy != MVT::v4i16)
1931 return Register();
1932 if ((Subtarget->hasNEON())) {
1933 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv4i16, RC: &ARM::DPRRegClass, Op0);
1934 }
1935 return Register();
1936}
1937
1938Register fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, Register Op0) {
1939 if (RetVT.SimpleTy != MVT::v8i16)
1940 return Register();
1941 if ((Subtarget->hasMVEIntegerOps())) {
1942 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLZs16, RC: &ARM::MQPRRegClass, Op0);
1943 }
1944 if ((Subtarget->hasNEON())) {
1945 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv8i16, RC: &ARM::QPRRegClass, Op0);
1946 }
1947 return Register();
1948}
1949
1950Register fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, Register Op0) {
1951 if (RetVT.SimpleTy != MVT::v2i32)
1952 return Register();
1953 if ((Subtarget->hasNEON())) {
1954 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv2i32, RC: &ARM::DPRRegClass, Op0);
1955 }
1956 return Register();
1957}
1958
1959Register fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, Register Op0) {
1960 if (RetVT.SimpleTy != MVT::v4i32)
1961 return Register();
1962 if ((Subtarget->hasMVEIntegerOps())) {
1963 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLZs32, RC: &ARM::MQPRRegClass, Op0);
1964 }
1965 if ((Subtarget->hasNEON())) {
1966 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv4i32, RC: &ARM::QPRRegClass, Op0);
1967 }
1968 return Register();
1969}
1970
1971Register fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, Register Op0) {
1972 switch (VT.SimpleTy) {
1973 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
1974 case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0);
1975 case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0);
1976 case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0);
1977 case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0);
1978 case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0);
1979 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
1980 default: return Register();
1981 }
1982}
1983
1984// FastEmit functions for ISD::CTPOP.
1985
1986Register fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, Register Op0) {
1987 if (RetVT.SimpleTy != MVT::v8i8)
1988 return Register();
1989 if ((Subtarget->hasNEON())) {
1990 return fastEmitInst_r(MachineInstOpcode: ARM::VCNTd, RC: &ARM::DPRRegClass, Op0);
1991 }
1992 return Register();
1993}
1994
1995Register fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, Register Op0) {
1996 if (RetVT.SimpleTy != MVT::v16i8)
1997 return Register();
1998 if ((Subtarget->hasNEON())) {
1999 return fastEmitInst_r(MachineInstOpcode: ARM::VCNTq, RC: &ARM::QPRRegClass, Op0);
2000 }
2001 return Register();
2002}
2003
2004Register fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, Register Op0) {
2005 switch (VT.SimpleTy) {
2006 case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0);
2007 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
2008 default: return Register();
2009 }
2010}
2011
2012// FastEmit functions for ISD::FABS.
2013
2014Register fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, Register Op0) {
2015 if (RetVT.SimpleTy != MVT::f16)
2016 return Register();
2017 if ((Subtarget->hasFullFP16())) {
2018 return fastEmitInst_r(MachineInstOpcode: ARM::VABSH, RC: &ARM::HPRRegClass, Op0);
2019 }
2020 return Register();
2021}
2022
2023Register fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, Register Op0) {
2024 if (RetVT.SimpleTy != MVT::f32)
2025 return Register();
2026 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
2027 return fastEmitInst_r(MachineInstOpcode: ARM::VABSS, RC: &ARM::SPRRegClass, Op0);
2028 }
2029 return Register();
2030}
2031
2032Register fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, Register Op0) {
2033 if (RetVT.SimpleTy != MVT::f64)
2034 return Register();
2035 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2036 return fastEmitInst_r(MachineInstOpcode: ARM::VABSD, RC: &ARM::DPRRegClass, Op0);
2037 }
2038 return Register();
2039}
2040
2041Register fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, Register Op0) {
2042 if (RetVT.SimpleTy != MVT::v4f16)
2043 return Register();
2044 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2045 return fastEmitInst_r(MachineInstOpcode: ARM::VABShd, RC: &ARM::DPRRegClass, Op0);
2046 }
2047 return Register();
2048}
2049
2050Register fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, Register Op0) {
2051 if (RetVT.SimpleTy != MVT::v8f16)
2052 return Register();
2053 if ((Subtarget->hasMVEIntegerOps())) {
2054 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSf16, RC: &ARM::MQPRRegClass, Op0);
2055 }
2056 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2057 return fastEmitInst_r(MachineInstOpcode: ARM::VABShq, RC: &ARM::QPRRegClass, Op0);
2058 }
2059 return Register();
2060}
2061
2062Register fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, Register Op0) {
2063 if (RetVT.SimpleTy != MVT::v2f32)
2064 return Register();
2065 if ((Subtarget->hasNEON())) {
2066 return fastEmitInst_r(MachineInstOpcode: ARM::VABSfd, RC: &ARM::DPRRegClass, Op0);
2067 }
2068 return Register();
2069}
2070
2071Register fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, Register Op0) {
2072 if (RetVT.SimpleTy != MVT::v4f32)
2073 return Register();
2074 if ((Subtarget->hasMVEIntegerOps())) {
2075 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSf32, RC: &ARM::MQPRRegClass, Op0);
2076 }
2077 if ((Subtarget->hasNEON())) {
2078 return fastEmitInst_r(MachineInstOpcode: ARM::VABSfq, RC: &ARM::QPRRegClass, Op0);
2079 }
2080 return Register();
2081}
2082
2083Register fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, Register Op0) {
2084 switch (VT.SimpleTy) {
2085 case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0);
2086 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
2087 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
2088 case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0);
2089 case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0);
2090 case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0);
2091 case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0);
2092 default: return Register();
2093 }
2094}
2095
2096// FastEmit functions for ISD::FCEIL.
2097
2098Register fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
2099 if (RetVT.SimpleTy != MVT::f16)
2100 return Register();
2101 if ((Subtarget->hasFullFP16())) {
2102 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPH, RC: &ARM::HPRRegClass, Op0);
2103 }
2104 return Register();
2105}
2106
2107Register fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
2108 if (RetVT.SimpleTy != MVT::f32)
2109 return Register();
2110 if ((Subtarget->hasFPARMv8Base())) {
2111 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPS, RC: &ARM::SPRRegClass, Op0);
2112 }
2113 return Register();
2114}
2115
2116Register fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
2117 if (RetVT.SimpleTy != MVT::f64)
2118 return Register();
2119 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2120 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPD, RC: &ARM::DPRRegClass, Op0);
2121 }
2122 return Register();
2123}
2124
2125Register fastEmit_ISD_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
2126 if (RetVT.SimpleTy != MVT::v4f16)
2127 return Register();
2128 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2129 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNDh, RC: &ARM::DPRRegClass, Op0);
2130 }
2131 return Register();
2132}
2133
2134Register fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
2135 if (RetVT.SimpleTy != MVT::v8f16)
2136 return Register();
2137 if ((Subtarget->hasMVEFloatOps())) {
2138 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16P, RC: &ARM::MQPRRegClass, Op0);
2139 }
2140 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2141 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNQh, RC: &ARM::QPRRegClass, Op0);
2142 }
2143 return Register();
2144}
2145
2146Register fastEmit_ISD_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
2147 if (RetVT.SimpleTy != MVT::v2f32)
2148 return Register();
2149 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2150 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNDf, RC: &ARM::DPRRegClass, Op0);
2151 }
2152 return Register();
2153}
2154
2155Register fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
2156 if (RetVT.SimpleTy != MVT::v4f32)
2157 return Register();
2158 if ((Subtarget->hasMVEFloatOps())) {
2159 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32P, RC: &ARM::MQPRRegClass, Op0);
2160 }
2161 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2162 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNQf, RC: &ARM::QPRRegClass, Op0);
2163 }
2164 return Register();
2165}
2166
2167Register fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
2168 switch (VT.SimpleTy) {
2169 case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0);
2170 case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0);
2171 case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0);
2172 case MVT::v4f16: return fastEmit_ISD_FCEIL_MVT_v4f16_r(RetVT, Op0);
2173 case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0);
2174 case MVT::v2f32: return fastEmit_ISD_FCEIL_MVT_v2f32_r(RetVT, Op0);
2175 case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0);
2176 default: return Register();
2177 }
2178}
2179
2180// FastEmit functions for ISD::FFLOOR.
2181
2182Register fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
2183 if (RetVT.SimpleTy != MVT::f16)
2184 return Register();
2185 if ((Subtarget->hasFullFP16())) {
2186 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMH, RC: &ARM::HPRRegClass, Op0);
2187 }
2188 return Register();
2189}
2190
2191Register fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
2192 if (RetVT.SimpleTy != MVT::f32)
2193 return Register();
2194 if ((Subtarget->hasFPARMv8Base())) {
2195 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMS, RC: &ARM::SPRRegClass, Op0);
2196 }
2197 return Register();
2198}
2199
2200Register fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
2201 if (RetVT.SimpleTy != MVT::f64)
2202 return Register();
2203 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2204 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMD, RC: &ARM::DPRRegClass, Op0);
2205 }
2206 return Register();
2207}
2208
2209Register fastEmit_ISD_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
2210 if (RetVT.SimpleTy != MVT::v4f16)
2211 return Register();
2212 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2213 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNDh, RC: &ARM::DPRRegClass, Op0);
2214 }
2215 return Register();
2216}
2217
2218Register fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
2219 if (RetVT.SimpleTy != MVT::v8f16)
2220 return Register();
2221 if ((Subtarget->hasMVEFloatOps())) {
2222 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16M, RC: &ARM::MQPRRegClass, Op0);
2223 }
2224 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2225 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNQh, RC: &ARM::QPRRegClass, Op0);
2226 }
2227 return Register();
2228}
2229
2230Register fastEmit_ISD_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
2231 if (RetVT.SimpleTy != MVT::v2f32)
2232 return Register();
2233 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2234 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNDf, RC: &ARM::DPRRegClass, Op0);
2235 }
2236 return Register();
2237}
2238
2239Register fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
2240 if (RetVT.SimpleTy != MVT::v4f32)
2241 return Register();
2242 if ((Subtarget->hasMVEFloatOps())) {
2243 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32M, RC: &ARM::MQPRRegClass, Op0);
2244 }
2245 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2246 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNQf, RC: &ARM::QPRRegClass, Op0);
2247 }
2248 return Register();
2249}
2250
2251Register fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
2252 switch (VT.SimpleTy) {
2253 case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0);
2254 case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0);
2255 case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0);
2256 case MVT::v4f16: return fastEmit_ISD_FFLOOR_MVT_v4f16_r(RetVT, Op0);
2257 case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0);
2258 case MVT::v2f32: return fastEmit_ISD_FFLOOR_MVT_v2f32_r(RetVT, Op0);
2259 case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0);
2260 default: return Register();
2261 }
2262}
2263
2264// FastEmit functions for ISD::FNEARBYINT.
2265
2266Register fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
2267 if (RetVT.SimpleTy != MVT::f16)
2268 return Register();
2269 if ((Subtarget->hasFullFP16())) {
2270 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRH, RC: &ARM::HPRRegClass, Op0);
2271 }
2272 return Register();
2273}
2274
2275Register fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
2276 if (RetVT.SimpleTy != MVT::f32)
2277 return Register();
2278 if ((Subtarget->hasFPARMv8Base())) {
2279 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRS, RC: &ARM::SPRRegClass, Op0);
2280 }
2281 return Register();
2282}
2283
2284Register fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
2285 if (RetVT.SimpleTy != MVT::f64)
2286 return Register();
2287 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2288 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRD, RC: &ARM::DPRRegClass, Op0);
2289 }
2290 return Register();
2291}
2292
2293Register fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
2294 switch (VT.SimpleTy) {
2295 case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0);
2296 case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0);
2297 case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0);
2298 default: return Register();
2299 }
2300}
2301
2302// FastEmit functions for ISD::FNEG.
2303
2304Register fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, Register Op0) {
2305 if (RetVT.SimpleTy != MVT::f16)
2306 return Register();
2307 if ((Subtarget->hasFullFP16())) {
2308 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGH, RC: &ARM::HPRRegClass, Op0);
2309 }
2310 return Register();
2311}
2312
2313Register fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, Register Op0) {
2314 if (RetVT.SimpleTy != MVT::f32)
2315 return Register();
2316 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
2317 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGS, RC: &ARM::SPRRegClass, Op0);
2318 }
2319 return Register();
2320}
2321
2322Register fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, Register Op0) {
2323 if (RetVT.SimpleTy != MVT::f64)
2324 return Register();
2325 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2326 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGD, RC: &ARM::DPRRegClass, Op0);
2327 }
2328 return Register();
2329}
2330
2331Register fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, Register Op0) {
2332 if (RetVT.SimpleTy != MVT::v4f16)
2333 return Register();
2334 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2335 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGhd, RC: &ARM::DPRRegClass, Op0);
2336 }
2337 return Register();
2338}
2339
2340Register fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, Register Op0) {
2341 if (RetVT.SimpleTy != MVT::v8f16)
2342 return Register();
2343 if ((Subtarget->hasMVEIntegerOps())) {
2344 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VNEGf16, RC: &ARM::MQPRRegClass, Op0);
2345 }
2346 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2347 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGhq, RC: &ARM::QPRRegClass, Op0);
2348 }
2349 return Register();
2350}
2351
2352Register fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, Register Op0) {
2353 if (RetVT.SimpleTy != MVT::v2f32)
2354 return Register();
2355 if ((Subtarget->hasNEON())) {
2356 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGfd, RC: &ARM::DPRRegClass, Op0);
2357 }
2358 return Register();
2359}
2360
2361Register fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, Register Op0) {
2362 if (RetVT.SimpleTy != MVT::v4f32)
2363 return Register();
2364 if ((Subtarget->hasMVEIntegerOps())) {
2365 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VNEGf32, RC: &ARM::MQPRRegClass, Op0);
2366 }
2367 if ((Subtarget->hasNEON())) {
2368 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGf32q, RC: &ARM::QPRRegClass, Op0);
2369 }
2370 return Register();
2371}
2372
2373Register fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, Register Op0) {
2374 switch (VT.SimpleTy) {
2375 case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0);
2376 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
2377 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
2378 case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0);
2379 case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0);
2380 case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0);
2381 case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0);
2382 default: return Register();
2383 }
2384}
2385
2386// FastEmit functions for ISD::FP_EXTEND.
2387
2388Register fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
2389 if (RetVT.SimpleTy != MVT::f64)
2390 return Register();
2391 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2392 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTDS, RC: &ARM::DPRRegClass, Op0);
2393 }
2394 return Register();
2395}
2396
2397Register fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
2398 if (RetVT.SimpleTy != MVT::v4f32)
2399 return Register();
2400 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2f, RC: &ARM::QPRRegClass, Op0);
2401}
2402
2403Register fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
2404 switch (VT.SimpleTy) {
2405 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
2406 case MVT::v4f16: return fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
2407 default: return Register();
2408 }
2409}
2410
2411// FastEmit functions for ISD::FP_ROUND.
2412
2413Register fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
2414 if (RetVT.SimpleTy != MVT::f32)
2415 return Register();
2416 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2417 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTSD, RC: &ARM::SPRRegClass, Op0);
2418 }
2419 return Register();
2420}
2421
2422Register fastEmit_ISD_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
2423 if (RetVT.SimpleTy != MVT::v4f16)
2424 return Register();
2425 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2h, RC: &ARM::DPRRegClass, Op0);
2426}
2427
2428Register fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
2429 switch (VT.SimpleTy) {
2430 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
2431 case MVT::v4f32: return fastEmit_ISD_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
2432 default: return Register();
2433 }
2434}
2435
2436// FastEmit functions for ISD::FP_TO_SINT.
2437
2438Register fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2439 if (RetVT.SimpleTy != MVT::v4i16)
2440 return Register();
2441 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2442 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2sd, RC: &ARM::DPRRegClass, Op0);
2443 }
2444 return Register();
2445}
2446
2447Register fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2448 if (RetVT.SimpleTy != MVT::v8i16)
2449 return Register();
2450 if ((Subtarget->hasMVEFloatOps())) {
2451 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTs16f16z, RC: &ARM::MQPRRegClass, Op0);
2452 }
2453 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2454 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2sq, RC: &ARM::QPRRegClass, Op0);
2455 }
2456 return Register();
2457}
2458
2459Register fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2460 if (RetVT.SimpleTy != MVT::v2i32)
2461 return Register();
2462 if ((Subtarget->hasNEON())) {
2463 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2sd, RC: &ARM::DPRRegClass, Op0);
2464 }
2465 return Register();
2466}
2467
2468Register fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2469 if (RetVT.SimpleTy != MVT::v4i32)
2470 return Register();
2471 if ((Subtarget->hasMVEFloatOps())) {
2472 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTs32f32z, RC: &ARM::MQPRRegClass, Op0);
2473 }
2474 if ((Subtarget->hasNEON())) {
2475 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2sq, RC: &ARM::QPRRegClass, Op0);
2476 }
2477 return Register();
2478}
2479
2480Register fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
2481 switch (VT.SimpleTy) {
2482 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
2483 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
2484 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
2485 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
2486 default: return Register();
2487 }
2488}
2489
2490// FastEmit functions for ISD::FP_TO_UINT.
2491
2492Register fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2493 if (RetVT.SimpleTy != MVT::v4i16)
2494 return Register();
2495 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2496 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2ud, RC: &ARM::DPRRegClass, Op0);
2497 }
2498 return Register();
2499}
2500
2501Register fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2502 if (RetVT.SimpleTy != MVT::v8i16)
2503 return Register();
2504 if ((Subtarget->hasMVEFloatOps())) {
2505 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTu16f16z, RC: &ARM::MQPRRegClass, Op0);
2506 }
2507 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2508 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2uq, RC: &ARM::QPRRegClass, Op0);
2509 }
2510 return Register();
2511}
2512
2513Register fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2514 if (RetVT.SimpleTy != MVT::v2i32)
2515 return Register();
2516 if ((Subtarget->hasNEON())) {
2517 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2ud, RC: &ARM::DPRRegClass, Op0);
2518 }
2519 return Register();
2520}
2521
2522Register fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2523 if (RetVT.SimpleTy != MVT::v4i32)
2524 return Register();
2525 if ((Subtarget->hasMVEFloatOps())) {
2526 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTu32f32z, RC: &ARM::MQPRRegClass, Op0);
2527 }
2528 if ((Subtarget->hasNEON())) {
2529 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2uq, RC: &ARM::QPRRegClass, Op0);
2530 }
2531 return Register();
2532}
2533
2534Register fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
2535 switch (VT.SimpleTy) {
2536 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
2537 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
2538 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
2539 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
2540 default: return Register();
2541 }
2542}
2543
2544// FastEmit functions for ISD::FRINT.
2545
2546Register fastEmit_ISD_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
2547 if (RetVT.SimpleTy != MVT::f16)
2548 return Register();
2549 if ((Subtarget->hasFullFP16())) {
2550 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXH, RC: &ARM::HPRRegClass, Op0);
2551 }
2552 return Register();
2553}
2554
2555Register fastEmit_ISD_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
2556 if (RetVT.SimpleTy != MVT::f32)
2557 return Register();
2558 if ((Subtarget->hasFPARMv8Base())) {
2559 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXS, RC: &ARM::SPRRegClass, Op0);
2560 }
2561 return Register();
2562}
2563
2564Register fastEmit_ISD_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
2565 if (RetVT.SimpleTy != MVT::f64)
2566 return Register();
2567 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2568 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXD, RC: &ARM::DPRRegClass, Op0);
2569 }
2570 return Register();
2571}
2572
2573Register fastEmit_ISD_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2574 if (RetVT.SimpleTy != MVT::v4f16)
2575 return Register();
2576 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2577 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNDh, RC: &ARM::DPRRegClass, Op0);
2578 }
2579 return Register();
2580}
2581
2582Register fastEmit_ISD_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2583 if (RetVT.SimpleTy != MVT::v8f16)
2584 return Register();
2585 if ((Subtarget->hasMVEFloatOps())) {
2586 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16X, RC: &ARM::MQPRRegClass, Op0);
2587 }
2588 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2589 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNQh, RC: &ARM::QPRRegClass, Op0);
2590 }
2591 return Register();
2592}
2593
2594Register fastEmit_ISD_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2595 if (RetVT.SimpleTy != MVT::v2f32)
2596 return Register();
2597 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2598 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNDf, RC: &ARM::DPRRegClass, Op0);
2599 }
2600 return Register();
2601}
2602
2603Register fastEmit_ISD_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2604 if (RetVT.SimpleTy != MVT::v4f32)
2605 return Register();
2606 if ((Subtarget->hasMVEFloatOps())) {
2607 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32X, RC: &ARM::MQPRRegClass, Op0);
2608 }
2609 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2610 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNQf, RC: &ARM::QPRRegClass, Op0);
2611 }
2612 return Register();
2613}
2614
2615Register fastEmit_ISD_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
2616 switch (VT.SimpleTy) {
2617 case MVT::f16: return fastEmit_ISD_FRINT_MVT_f16_r(RetVT, Op0);
2618 case MVT::f32: return fastEmit_ISD_FRINT_MVT_f32_r(RetVT, Op0);
2619 case MVT::f64: return fastEmit_ISD_FRINT_MVT_f64_r(RetVT, Op0);
2620 case MVT::v4f16: return fastEmit_ISD_FRINT_MVT_v4f16_r(RetVT, Op0);
2621 case MVT::v8f16: return fastEmit_ISD_FRINT_MVT_v8f16_r(RetVT, Op0);
2622 case MVT::v2f32: return fastEmit_ISD_FRINT_MVT_v2f32_r(RetVT, Op0);
2623 case MVT::v4f32: return fastEmit_ISD_FRINT_MVT_v4f32_r(RetVT, Op0);
2624 default: return Register();
2625 }
2626}
2627
2628// FastEmit functions for ISD::FROUND.
2629
2630Register fastEmit_ISD_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
2631 if (RetVT.SimpleTy != MVT::f16)
2632 return Register();
2633 if ((Subtarget->hasFullFP16())) {
2634 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAH, RC: &ARM::HPRRegClass, Op0);
2635 }
2636 return Register();
2637}
2638
2639Register fastEmit_ISD_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
2640 if (RetVT.SimpleTy != MVT::f32)
2641 return Register();
2642 if ((Subtarget->hasFPARMv8Base())) {
2643 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAS, RC: &ARM::SPRRegClass, Op0);
2644 }
2645 return Register();
2646}
2647
2648Register fastEmit_ISD_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
2649 if (RetVT.SimpleTy != MVT::f64)
2650 return Register();
2651 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2652 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAD, RC: &ARM::DPRRegClass, Op0);
2653 }
2654 return Register();
2655}
2656
2657Register fastEmit_ISD_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
2658 if (RetVT.SimpleTy != MVT::v4f16)
2659 return Register();
2660 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2661 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANDh, RC: &ARM::DPRRegClass, Op0);
2662 }
2663 return Register();
2664}
2665
2666Register fastEmit_ISD_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
2667 if (RetVT.SimpleTy != MVT::v8f16)
2668 return Register();
2669 if ((Subtarget->hasMVEFloatOps())) {
2670 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16A, RC: &ARM::MQPRRegClass, Op0);
2671 }
2672 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2673 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANQh, RC: &ARM::QPRRegClass, Op0);
2674 }
2675 return Register();
2676}
2677
2678Register fastEmit_ISD_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
2679 if (RetVT.SimpleTy != MVT::v2f32)
2680 return Register();
2681 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2682 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANDf, RC: &ARM::DPRRegClass, Op0);
2683 }
2684 return Register();
2685}
2686
2687Register fastEmit_ISD_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
2688 if (RetVT.SimpleTy != MVT::v4f32)
2689 return Register();
2690 if ((Subtarget->hasMVEFloatOps())) {
2691 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32A, RC: &ARM::MQPRRegClass, Op0);
2692 }
2693 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2694 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANQf, RC: &ARM::QPRRegClass, Op0);
2695 }
2696 return Register();
2697}
2698
2699Register fastEmit_ISD_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
2700 switch (VT.SimpleTy) {
2701 case MVT::f16: return fastEmit_ISD_FROUND_MVT_f16_r(RetVT, Op0);
2702 case MVT::f32: return fastEmit_ISD_FROUND_MVT_f32_r(RetVT, Op0);
2703 case MVT::f64: return fastEmit_ISD_FROUND_MVT_f64_r(RetVT, Op0);
2704 case MVT::v4f16: return fastEmit_ISD_FROUND_MVT_v4f16_r(RetVT, Op0);
2705 case MVT::v8f16: return fastEmit_ISD_FROUND_MVT_v8f16_r(RetVT, Op0);
2706 case MVT::v2f32: return fastEmit_ISD_FROUND_MVT_v2f32_r(RetVT, Op0);
2707 case MVT::v4f32: return fastEmit_ISD_FROUND_MVT_v4f32_r(RetVT, Op0);
2708 default: return Register();
2709 }
2710}
2711
2712// FastEmit functions for ISD::FROUNDEVEN.
2713
2714Register fastEmit_ISD_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
2715 if (RetVT.SimpleTy != MVT::f16)
2716 return Register();
2717 if ((Subtarget->hasFullFP16())) {
2718 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNH, RC: &ARM::HPRRegClass, Op0);
2719 }
2720 return Register();
2721}
2722
2723Register fastEmit_ISD_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
2724 if (RetVT.SimpleTy != MVT::f32)
2725 return Register();
2726 if ((Subtarget->hasFPARMv8Base())) {
2727 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNS, RC: &ARM::SPRRegClass, Op0);
2728 }
2729 return Register();
2730}
2731
2732Register fastEmit_ISD_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
2733 if (RetVT.SimpleTy != MVT::f64)
2734 return Register();
2735 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2736 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTND, RC: &ARM::DPRRegClass, Op0);
2737 }
2738 return Register();
2739}
2740
2741Register fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
2742 if (RetVT.SimpleTy != MVT::v4f16)
2743 return Register();
2744 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2745 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNDh, RC: &ARM::DPRRegClass, Op0);
2746 }
2747 return Register();
2748}
2749
2750Register fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
2751 if (RetVT.SimpleTy != MVT::v8f16)
2752 return Register();
2753 if ((Subtarget->hasMVEFloatOps())) {
2754 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16N, RC: &ARM::MQPRRegClass, Op0);
2755 }
2756 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2757 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNQh, RC: &ARM::QPRRegClass, Op0);
2758 }
2759 return Register();
2760}
2761
2762Register fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
2763 if (RetVT.SimpleTy != MVT::v2f32)
2764 return Register();
2765 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2766 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNDf, RC: &ARM::DPRRegClass, Op0);
2767 }
2768 return Register();
2769}
2770
2771Register fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
2772 if (RetVT.SimpleTy != MVT::v4f32)
2773 return Register();
2774 if ((Subtarget->hasMVEFloatOps())) {
2775 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32N, RC: &ARM::MQPRRegClass, Op0);
2776 }
2777 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2778 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNQf, RC: &ARM::QPRRegClass, Op0);
2779 }
2780 return Register();
2781}
2782
2783Register fastEmit_ISD_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
2784 switch (VT.SimpleTy) {
2785 case MVT::f16: return fastEmit_ISD_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
2786 case MVT::f32: return fastEmit_ISD_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
2787 case MVT::f64: return fastEmit_ISD_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
2788 case MVT::v4f16: return fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
2789 case MVT::v8f16: return fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
2790 case MVT::v2f32: return fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
2791 case MVT::v4f32: return fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
2792 default: return Register();
2793 }
2794}
2795
2796// FastEmit functions for ISD::FSQRT.
2797
2798Register fastEmit_ISD_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
2799 if (RetVT.SimpleTy != MVT::f16)
2800 return Register();
2801 if ((Subtarget->hasFullFP16())) {
2802 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTH, RC: &ARM::HPRRegClass, Op0);
2803 }
2804 return Register();
2805}
2806
2807Register fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
2808 if (RetVT.SimpleTy != MVT::f32)
2809 return Register();
2810 if ((Subtarget->hasVFP2Base())) {
2811 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTS, RC: &ARM::SPRRegClass, Op0);
2812 }
2813 return Register();
2814}
2815
2816Register fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
2817 if (RetVT.SimpleTy != MVT::f64)
2818 return Register();
2819 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2820 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTD, RC: &ARM::DPRRegClass, Op0);
2821 }
2822 return Register();
2823}
2824
2825Register fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
2826 switch (VT.SimpleTy) {
2827 case MVT::f16: return fastEmit_ISD_FSQRT_MVT_f16_r(RetVT, Op0);
2828 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
2829 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
2830 default: return Register();
2831 }
2832}
2833
2834// FastEmit functions for ISD::FTRUNC.
2835
2836Register fastEmit_ISD_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
2837 if (RetVT.SimpleTy != MVT::f16)
2838 return Register();
2839 if ((Subtarget->hasFullFP16())) {
2840 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZH, RC: &ARM::HPRRegClass, Op0);
2841 }
2842 return Register();
2843}
2844
2845Register fastEmit_ISD_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
2846 if (RetVT.SimpleTy != MVT::f32)
2847 return Register();
2848 if ((Subtarget->hasFPARMv8Base())) {
2849 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZS, RC: &ARM::SPRRegClass, Op0);
2850 }
2851 return Register();
2852}
2853
2854Register fastEmit_ISD_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
2855 if (RetVT.SimpleTy != MVT::f64)
2856 return Register();
2857 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2858 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZD, RC: &ARM::DPRRegClass, Op0);
2859 }
2860 return Register();
2861}
2862
2863Register fastEmit_ISD_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
2864 if (RetVT.SimpleTy != MVT::v4f16)
2865 return Register();
2866 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2867 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNDh, RC: &ARM::DPRRegClass, Op0);
2868 }
2869 return Register();
2870}
2871
2872Register fastEmit_ISD_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
2873 if (RetVT.SimpleTy != MVT::v8f16)
2874 return Register();
2875 if ((Subtarget->hasMVEFloatOps())) {
2876 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16Z, RC: &ARM::MQPRRegClass, Op0);
2877 }
2878 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2879 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNQh, RC: &ARM::QPRRegClass, Op0);
2880 }
2881 return Register();
2882}
2883
2884Register fastEmit_ISD_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
2885 if (RetVT.SimpleTy != MVT::v2f32)
2886 return Register();
2887 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2888 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNDf, RC: &ARM::DPRRegClass, Op0);
2889 }
2890 return Register();
2891}
2892
2893Register fastEmit_ISD_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
2894 if (RetVT.SimpleTy != MVT::v4f32)
2895 return Register();
2896 if ((Subtarget->hasMVEFloatOps())) {
2897 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32Z, RC: &ARM::MQPRRegClass, Op0);
2898 }
2899 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2900 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNQf, RC: &ARM::QPRRegClass, Op0);
2901 }
2902 return Register();
2903}
2904
2905Register fastEmit_ISD_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
2906 switch (VT.SimpleTy) {
2907 case MVT::f16: return fastEmit_ISD_FTRUNC_MVT_f16_r(RetVT, Op0);
2908 case MVT::f32: return fastEmit_ISD_FTRUNC_MVT_f32_r(RetVT, Op0);
2909 case MVT::f64: return fastEmit_ISD_FTRUNC_MVT_f64_r(RetVT, Op0);
2910 case MVT::v4f16: return fastEmit_ISD_FTRUNC_MVT_v4f16_r(RetVT, Op0);
2911 case MVT::v8f16: return fastEmit_ISD_FTRUNC_MVT_v8f16_r(RetVT, Op0);
2912 case MVT::v2f32: return fastEmit_ISD_FTRUNC_MVT_v2f32_r(RetVT, Op0);
2913 case MVT::v4f32: return fastEmit_ISD_FTRUNC_MVT_v4f32_r(RetVT, Op0);
2914 default: return Register();
2915 }
2916}
2917
2918// FastEmit functions for ISD::SIGN_EXTEND.
2919
2920Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i8_r(MVT RetVT, Register Op0) {
2921 if (RetVT.SimpleTy != MVT::v8i16)
2922 return Register();
2923 if ((Subtarget->hasNEON())) {
2924 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLsv8i16, RC: &ARM::QPRRegClass, Op0);
2925 }
2926 return Register();
2927}
2928
2929Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i16_r(MVT RetVT, Register Op0) {
2930 if (RetVT.SimpleTy != MVT::v4i32)
2931 return Register();
2932 if ((Subtarget->hasNEON())) {
2933 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLsv4i32, RC: &ARM::QPRRegClass, Op0);
2934 }
2935 return Register();
2936}
2937
2938Register fastEmit_ISD_SIGN_EXTEND_MVT_v2i32_r(MVT RetVT, Register Op0) {
2939 if (RetVT.SimpleTy != MVT::v2i64)
2940 return Register();
2941 if ((Subtarget->hasNEON())) {
2942 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLsv2i64, RC: &ARM::QPRRegClass, Op0);
2943 }
2944 return Register();
2945}
2946
2947Register fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
2948 switch (VT.SimpleTy) {
2949 case MVT::v8i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i8_r(RetVT, Op0);
2950 case MVT::v4i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i16_r(RetVT, Op0);
2951 case MVT::v2i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i32_r(RetVT, Op0);
2952 default: return Register();
2953 }
2954}
2955
2956// FastEmit functions for ISD::SINT_TO_FP.
2957
2958Register fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
2959 if (RetVT.SimpleTy != MVT::v4f16)
2960 return Register();
2961 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2962 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2hd, RC: &ARM::DPRRegClass, Op0);
2963 }
2964 return Register();
2965}
2966
2967Register fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
2968 if (RetVT.SimpleTy != MVT::v8f16)
2969 return Register();
2970 if ((Subtarget->hasMVEFloatOps())) {
2971 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf16s16n, RC: &ARM::MQPRRegClass, Op0);
2972 }
2973 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2974 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2hq, RC: &ARM::QPRRegClass, Op0);
2975 }
2976 return Register();
2977}
2978
2979Register fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
2980 if (RetVT.SimpleTy != MVT::v2f32)
2981 return Register();
2982 if ((Subtarget->hasNEON())) {
2983 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2fd, RC: &ARM::DPRRegClass, Op0);
2984 }
2985 return Register();
2986}
2987
2988Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2989 if (RetVT.SimpleTy != MVT::v4f32)
2990 return Register();
2991 if ((Subtarget->hasMVEFloatOps())) {
2992 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf32s32n, RC: &ARM::MQPRRegClass, Op0);
2993 }
2994 if ((Subtarget->hasNEON())) {
2995 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2fq, RC: &ARM::QPRRegClass, Op0);
2996 }
2997 return Register();
2998}
2999
3000Register fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3001 switch (VT.SimpleTy) {
3002 case MVT::v4i16: return fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
3003 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3004 case MVT::v2i32: return fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
3005 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3006 default: return Register();
3007 }
3008}
3009
3010// FastEmit functions for ISD::STRICT_FCEIL.
3011
3012Register fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
3013 if (RetVT.SimpleTy != MVT::f16)
3014 return Register();
3015 if ((Subtarget->hasFullFP16())) {
3016 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPH, RC: &ARM::HPRRegClass, Op0);
3017 }
3018 return Register();
3019}
3020
3021Register fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
3022 if (RetVT.SimpleTy != MVT::f32)
3023 return Register();
3024 if ((Subtarget->hasFPARMv8Base())) {
3025 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPS, RC: &ARM::SPRRegClass, Op0);
3026 }
3027 return Register();
3028}
3029
3030Register fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
3031 if (RetVT.SimpleTy != MVT::f64)
3032 return Register();
3033 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3034 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPD, RC: &ARM::DPRRegClass, Op0);
3035 }
3036 return Register();
3037}
3038
3039Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
3040 if (RetVT.SimpleTy != MVT::v4f16)
3041 return Register();
3042 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3043 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNDh, RC: &ARM::DPRRegClass, Op0);
3044 }
3045 return Register();
3046}
3047
3048Register fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
3049 if (RetVT.SimpleTy != MVT::v8f16)
3050 return Register();
3051 if ((Subtarget->hasMVEFloatOps())) {
3052 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16P, RC: &ARM::MQPRRegClass, Op0);
3053 }
3054 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3055 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNQh, RC: &ARM::QPRRegClass, Op0);
3056 }
3057 return Register();
3058}
3059
3060Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
3061 if (RetVT.SimpleTy != MVT::v2f32)
3062 return Register();
3063 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3064 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNDf, RC: &ARM::DPRRegClass, Op0);
3065 }
3066 return Register();
3067}
3068
3069Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
3070 if (RetVT.SimpleTy != MVT::v4f32)
3071 return Register();
3072 if ((Subtarget->hasMVEFloatOps())) {
3073 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32P, RC: &ARM::MQPRRegClass, Op0);
3074 }
3075 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3076 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNQf, RC: &ARM::QPRRegClass, Op0);
3077 }
3078 return Register();
3079}
3080
3081Register fastEmit_ISD_STRICT_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
3082 switch (VT.SimpleTy) {
3083 case MVT::f16: return fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(RetVT, Op0);
3084 case MVT::f32: return fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(RetVT, Op0);
3085 case MVT::f64: return fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(RetVT, Op0);
3086 case MVT::v4f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(RetVT, Op0);
3087 case MVT::v8f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(RetVT, Op0);
3088 case MVT::v2f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(RetVT, Op0);
3089 case MVT::v4f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(RetVT, Op0);
3090 default: return Register();
3091 }
3092}
3093
3094// FastEmit functions for ISD::STRICT_FFLOOR.
3095
3096Register fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
3097 if (RetVT.SimpleTy != MVT::f16)
3098 return Register();
3099 if ((Subtarget->hasFullFP16())) {
3100 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMH, RC: &ARM::HPRRegClass, Op0);
3101 }
3102 return Register();
3103}
3104
3105Register fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
3106 if (RetVT.SimpleTy != MVT::f32)
3107 return Register();
3108 if ((Subtarget->hasFPARMv8Base())) {
3109 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMS, RC: &ARM::SPRRegClass, Op0);
3110 }
3111 return Register();
3112}
3113
3114Register fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
3115 if (RetVT.SimpleTy != MVT::f64)
3116 return Register();
3117 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3118 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMD, RC: &ARM::DPRRegClass, Op0);
3119 }
3120 return Register();
3121}
3122
3123Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
3124 if (RetVT.SimpleTy != MVT::v4f16)
3125 return Register();
3126 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3127 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNDh, RC: &ARM::DPRRegClass, Op0);
3128 }
3129 return Register();
3130}
3131
3132Register fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
3133 if (RetVT.SimpleTy != MVT::v8f16)
3134 return Register();
3135 if ((Subtarget->hasMVEFloatOps())) {
3136 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16M, RC: &ARM::MQPRRegClass, Op0);
3137 }
3138 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3139 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNQh, RC: &ARM::QPRRegClass, Op0);
3140 }
3141 return Register();
3142}
3143
3144Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
3145 if (RetVT.SimpleTy != MVT::v2f32)
3146 return Register();
3147 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3148 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNDf, RC: &ARM::DPRRegClass, Op0);
3149 }
3150 return Register();
3151}
3152
3153Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
3154 if (RetVT.SimpleTy != MVT::v4f32)
3155 return Register();
3156 if ((Subtarget->hasMVEFloatOps())) {
3157 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32M, RC: &ARM::MQPRRegClass, Op0);
3158 }
3159 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3160 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNQf, RC: &ARM::QPRRegClass, Op0);
3161 }
3162 return Register();
3163}
3164
3165Register fastEmit_ISD_STRICT_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
3166 switch (VT.SimpleTy) {
3167 case MVT::f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(RetVT, Op0);
3168 case MVT::f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(RetVT, Op0);
3169 case MVT::f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(RetVT, Op0);
3170 case MVT::v4f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(RetVT, Op0);
3171 case MVT::v8f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(RetVT, Op0);
3172 case MVT::v2f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(RetVT, Op0);
3173 case MVT::v4f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(RetVT, Op0);
3174 default: return Register();
3175 }
3176}
3177
3178// FastEmit functions for ISD::STRICT_FNEARBYINT.
3179
3180Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
3181 if (RetVT.SimpleTy != MVT::f16)
3182 return Register();
3183 if ((Subtarget->hasFullFP16())) {
3184 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRH, RC: &ARM::HPRRegClass, Op0);
3185 }
3186 return Register();
3187}
3188
3189Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
3190 if (RetVT.SimpleTy != MVT::f32)
3191 return Register();
3192 if ((Subtarget->hasFPARMv8Base())) {
3193 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRS, RC: &ARM::SPRRegClass, Op0);
3194 }
3195 return Register();
3196}
3197
3198Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
3199 if (RetVT.SimpleTy != MVT::f64)
3200 return Register();
3201 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3202 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRD, RC: &ARM::DPRRegClass, Op0);
3203 }
3204 return Register();
3205}
3206
3207Register fastEmit_ISD_STRICT_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
3208 switch (VT.SimpleTy) {
3209 case MVT::f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(RetVT, Op0);
3210 case MVT::f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(RetVT, Op0);
3211 case MVT::f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(RetVT, Op0);
3212 default: return Register();
3213 }
3214}
3215
3216// FastEmit functions for ISD::STRICT_FP_EXTEND.
3217
3218Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
3219 if (RetVT.SimpleTy != MVT::f64)
3220 return Register();
3221 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3222 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTDS, RC: &ARM::DPRRegClass, Op0);
3223 }
3224 return Register();
3225}
3226
3227Register fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3228 switch (VT.SimpleTy) {
3229 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0);
3230 default: return Register();
3231 }
3232}
3233
3234// FastEmit functions for ISD::STRICT_FP_ROUND.
3235
3236Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3237 if (RetVT.SimpleTy != MVT::f32)
3238 return Register();
3239 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3240 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTSD, RC: &ARM::SPRRegClass, Op0);
3241 }
3242 return Register();
3243}
3244
3245Register fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
3246 switch (VT.SimpleTy) {
3247 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0);
3248 default: return Register();
3249 }
3250}
3251
3252// FastEmit functions for ISD::STRICT_FRINT.
3253
3254Register fastEmit_ISD_STRICT_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
3255 if (RetVT.SimpleTy != MVT::f16)
3256 return Register();
3257 if ((Subtarget->hasFullFP16())) {
3258 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXH, RC: &ARM::HPRRegClass, Op0);
3259 }
3260 return Register();
3261}
3262
3263Register fastEmit_ISD_STRICT_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
3264 if (RetVT.SimpleTy != MVT::f32)
3265 return Register();
3266 if ((Subtarget->hasFPARMv8Base())) {
3267 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXS, RC: &ARM::SPRRegClass, Op0);
3268 }
3269 return Register();
3270}
3271
3272Register fastEmit_ISD_STRICT_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
3273 if (RetVT.SimpleTy != MVT::f64)
3274 return Register();
3275 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3276 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXD, RC: &ARM::DPRRegClass, Op0);
3277 }
3278 return Register();
3279}
3280
3281Register fastEmit_ISD_STRICT_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
3282 switch (VT.SimpleTy) {
3283 case MVT::f16: return fastEmit_ISD_STRICT_FRINT_MVT_f16_r(RetVT, Op0);
3284 case MVT::f32: return fastEmit_ISD_STRICT_FRINT_MVT_f32_r(RetVT, Op0);
3285 case MVT::f64: return fastEmit_ISD_STRICT_FRINT_MVT_f64_r(RetVT, Op0);
3286 default: return Register();
3287 }
3288}
3289
3290// FastEmit functions for ISD::STRICT_FROUND.
3291
3292Register fastEmit_ISD_STRICT_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
3293 if (RetVT.SimpleTy != MVT::f16)
3294 return Register();
3295 if ((Subtarget->hasFullFP16())) {
3296 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAH, RC: &ARM::HPRRegClass, Op0);
3297 }
3298 return Register();
3299}
3300
3301Register fastEmit_ISD_STRICT_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3302 if (RetVT.SimpleTy != MVT::f32)
3303 return Register();
3304 if ((Subtarget->hasFPARMv8Base())) {
3305 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAS, RC: &ARM::SPRRegClass, Op0);
3306 }
3307 return Register();
3308}
3309
3310Register fastEmit_ISD_STRICT_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3311 if (RetVT.SimpleTy != MVT::f64)
3312 return Register();
3313 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3314 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAD, RC: &ARM::DPRRegClass, Op0);
3315 }
3316 return Register();
3317}
3318
3319Register fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3320 if (RetVT.SimpleTy != MVT::v4f16)
3321 return Register();
3322 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3323 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANDh, RC: &ARM::DPRRegClass, Op0);
3324 }
3325 return Register();
3326}
3327
3328Register fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
3329 if (RetVT.SimpleTy != MVT::v8f16)
3330 return Register();
3331 if ((Subtarget->hasMVEFloatOps())) {
3332 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16A, RC: &ARM::MQPRRegClass, Op0);
3333 }
3334 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3335 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANQh, RC: &ARM::QPRRegClass, Op0);
3336 }
3337 return Register();
3338}
3339
3340Register fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3341 if (RetVT.SimpleTy != MVT::v2f32)
3342 return Register();
3343 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3344 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANDf, RC: &ARM::DPRRegClass, Op0);
3345 }
3346 return Register();
3347}
3348
3349Register fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3350 if (RetVT.SimpleTy != MVT::v4f32)
3351 return Register();
3352 if ((Subtarget->hasMVEFloatOps())) {
3353 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32A, RC: &ARM::MQPRRegClass, Op0);
3354 }
3355 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3356 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANQf, RC: &ARM::QPRRegClass, Op0);
3357 }
3358 return Register();
3359}
3360
3361Register fastEmit_ISD_STRICT_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
3362 switch (VT.SimpleTy) {
3363 case MVT::f16: return fastEmit_ISD_STRICT_FROUND_MVT_f16_r(RetVT, Op0);
3364 case MVT::f32: return fastEmit_ISD_STRICT_FROUND_MVT_f32_r(RetVT, Op0);
3365 case MVT::f64: return fastEmit_ISD_STRICT_FROUND_MVT_f64_r(RetVT, Op0);
3366 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(RetVT, Op0);
3367 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(RetVT, Op0);
3368 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(RetVT, Op0);
3369 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(RetVT, Op0);
3370 default: return Register();
3371 }
3372}
3373
3374// FastEmit functions for ISD::STRICT_FROUNDEVEN.
3375
3376Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
3377 if (RetVT.SimpleTy != MVT::f16)
3378 return Register();
3379 if ((Subtarget->hasFullFP16())) {
3380 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNH, RC: &ARM::HPRRegClass, Op0);
3381 }
3382 return Register();
3383}
3384
3385Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
3386 if (RetVT.SimpleTy != MVT::f32)
3387 return Register();
3388 if ((Subtarget->hasFPARMv8Base())) {
3389 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNS, RC: &ARM::SPRRegClass, Op0);
3390 }
3391 return Register();
3392}
3393
3394Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
3395 if (RetVT.SimpleTy != MVT::f64)
3396 return Register();
3397 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3398 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTND, RC: &ARM::DPRRegClass, Op0);
3399 }
3400 return Register();
3401}
3402
3403Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
3404 if (RetVT.SimpleTy != MVT::v4f16)
3405 return Register();
3406 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3407 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNDh, RC: &ARM::DPRRegClass, Op0);
3408 }
3409 return Register();
3410}
3411
3412Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
3413 if (RetVT.SimpleTy != MVT::v8f16)
3414 return Register();
3415 if ((Subtarget->hasMVEFloatOps())) {
3416 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16N, RC: &ARM::MQPRRegClass, Op0);
3417 }
3418 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3419 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNQh, RC: &ARM::QPRRegClass, Op0);
3420 }
3421 return Register();
3422}
3423
3424Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
3425 if (RetVT.SimpleTy != MVT::v2f32)
3426 return Register();
3427 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3428 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNDf, RC: &ARM::DPRRegClass, Op0);
3429 }
3430 return Register();
3431}
3432
3433Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
3434 if (RetVT.SimpleTy != MVT::v4f32)
3435 return Register();
3436 if ((Subtarget->hasMVEFloatOps())) {
3437 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32N, RC: &ARM::MQPRRegClass, Op0);
3438 }
3439 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3440 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNQf, RC: &ARM::QPRRegClass, Op0);
3441 }
3442 return Register();
3443}
3444
3445Register fastEmit_ISD_STRICT_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
3446 switch (VT.SimpleTy) {
3447 case MVT::f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
3448 case MVT::f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
3449 case MVT::f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
3450 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
3451 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
3452 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
3453 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
3454 default: return Register();
3455 }
3456}
3457
3458// FastEmit functions for ISD::STRICT_FSQRT.
3459
3460Register fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
3461 if (RetVT.SimpleTy != MVT::f16)
3462 return Register();
3463 if ((Subtarget->hasFullFP16())) {
3464 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTH, RC: &ARM::HPRRegClass, Op0);
3465 }
3466 return Register();
3467}
3468
3469Register fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
3470 if (RetVT.SimpleTy != MVT::f32)
3471 return Register();
3472 if ((Subtarget->hasVFP2Base())) {
3473 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTS, RC: &ARM::SPRRegClass, Op0);
3474 }
3475 return Register();
3476}
3477
3478Register fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
3479 if (RetVT.SimpleTy != MVT::f64)
3480 return Register();
3481 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3482 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTD, RC: &ARM::DPRRegClass, Op0);
3483 }
3484 return Register();
3485}
3486
3487Register fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
3488 switch (VT.SimpleTy) {
3489 case MVT::f16: return fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(RetVT, Op0);
3490 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0);
3491 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0);
3492 default: return Register();
3493 }
3494}
3495
3496// FastEmit functions for ISD::STRICT_FTRUNC.
3497
3498Register fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
3499 if (RetVT.SimpleTy != MVT::f16)
3500 return Register();
3501 if ((Subtarget->hasFullFP16())) {
3502 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZH, RC: &ARM::HPRRegClass, Op0);
3503 }
3504 return Register();
3505}
3506
3507Register fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
3508 if (RetVT.SimpleTy != MVT::f32)
3509 return Register();
3510 if ((Subtarget->hasFPARMv8Base())) {
3511 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZS, RC: &ARM::SPRRegClass, Op0);
3512 }
3513 return Register();
3514}
3515
3516Register fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
3517 if (RetVT.SimpleTy != MVT::f64)
3518 return Register();
3519 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3520 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZD, RC: &ARM::DPRRegClass, Op0);
3521 }
3522 return Register();
3523}
3524
3525Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
3526 if (RetVT.SimpleTy != MVT::v4f16)
3527 return Register();
3528 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3529 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNDh, RC: &ARM::DPRRegClass, Op0);
3530 }
3531 return Register();
3532}
3533
3534Register fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
3535 if (RetVT.SimpleTy != MVT::v8f16)
3536 return Register();
3537 if ((Subtarget->hasMVEFloatOps())) {
3538 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16Z, RC: &ARM::MQPRRegClass, Op0);
3539 }
3540 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3541 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNQh, RC: &ARM::QPRRegClass, Op0);
3542 }
3543 return Register();
3544}
3545
3546Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
3547 if (RetVT.SimpleTy != MVT::v2f32)
3548 return Register();
3549 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3550 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNDf, RC: &ARM::DPRRegClass, Op0);
3551 }
3552 return Register();
3553}
3554
3555Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
3556 if (RetVT.SimpleTy != MVT::v4f32)
3557 return Register();
3558 if ((Subtarget->hasMVEFloatOps())) {
3559 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32Z, RC: &ARM::MQPRRegClass, Op0);
3560 }
3561 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3562 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNQf, RC: &ARM::QPRRegClass, Op0);
3563 }
3564 return Register();
3565}
3566
3567Register fastEmit_ISD_STRICT_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
3568 switch (VT.SimpleTy) {
3569 case MVT::f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(RetVT, Op0);
3570 case MVT::f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(RetVT, Op0);
3571 case MVT::f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(RetVT, Op0);
3572 case MVT::v4f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(RetVT, Op0);
3573 case MVT::v8f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(RetVT, Op0);
3574 case MVT::v2f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(RetVT, Op0);
3575 case MVT::v4f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(RetVT, Op0);
3576 default: return Register();
3577 }
3578}
3579
3580// FastEmit functions for ISD::TRUNCATE.
3581
3582Register fastEmit_ISD_TRUNCATE_MVT_v8i16_r(MVT RetVT, Register Op0) {
3583 if (RetVT.SimpleTy != MVT::v8i8)
3584 return Register();
3585 if ((Subtarget->hasNEON())) {
3586 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVNv8i8, RC: &ARM::DPRRegClass, Op0);
3587 }
3588 return Register();
3589}
3590
3591Register fastEmit_ISD_TRUNCATE_MVT_v4i32_r(MVT RetVT, Register Op0) {
3592 if (RetVT.SimpleTy != MVT::v4i16)
3593 return Register();
3594 if ((Subtarget->hasNEON())) {
3595 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVNv4i16, RC: &ARM::DPRRegClass, Op0);
3596 }
3597 return Register();
3598}
3599
3600Register fastEmit_ISD_TRUNCATE_MVT_v2i64_r(MVT RetVT, Register Op0) {
3601 if (RetVT.SimpleTy != MVT::v2i32)
3602 return Register();
3603 if ((Subtarget->hasNEON())) {
3604 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVNv2i32, RC: &ARM::DPRRegClass, Op0);
3605 }
3606 return Register();
3607}
3608
3609Register fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, Register Op0) {
3610 switch (VT.SimpleTy) {
3611 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i16_r(RetVT, Op0);
3612 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_MVT_v4i32_r(RetVT, Op0);
3613 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_MVT_v2i64_r(RetVT, Op0);
3614 default: return Register();
3615 }
3616}
3617
3618// FastEmit functions for ISD::UINT_TO_FP.
3619
3620Register fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
3621 if (RetVT.SimpleTy != MVT::v4f16)
3622 return Register();
3623 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3624 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2hd, RC: &ARM::DPRRegClass, Op0);
3625 }
3626 return Register();
3627}
3628
3629Register fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
3630 if (RetVT.SimpleTy != MVT::v8f16)
3631 return Register();
3632 if ((Subtarget->hasMVEFloatOps())) {
3633 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf16u16n, RC: &ARM::MQPRRegClass, Op0);
3634 }
3635 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3636 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2hq, RC: &ARM::QPRRegClass, Op0);
3637 }
3638 return Register();
3639}
3640
3641Register fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
3642 if (RetVT.SimpleTy != MVT::v2f32)
3643 return Register();
3644 if ((Subtarget->hasNEON())) {
3645 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2fd, RC: &ARM::DPRRegClass, Op0);
3646 }
3647 return Register();
3648}
3649
3650Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
3651 if (RetVT.SimpleTy != MVT::v4f32)
3652 return Register();
3653 if ((Subtarget->hasMVEFloatOps())) {
3654 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf32u32n, RC: &ARM::MQPRRegClass, Op0);
3655 }
3656 if ((Subtarget->hasNEON())) {
3657 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2fq, RC: &ARM::QPRRegClass, Op0);
3658 }
3659 return Register();
3660}
3661
3662Register fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3663 switch (VT.SimpleTy) {
3664 case MVT::v4i16: return fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
3665 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3666 case MVT::v2i32: return fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
3667 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3668 default: return Register();
3669 }
3670}
3671
3672// FastEmit functions for ISD::VECREDUCE_ADD.
3673
3674Register fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(MVT RetVT, Register Op0) {
3675 if (RetVT.SimpleTy != MVT::i32)
3676 return Register();
3677 if ((Subtarget->hasMVEIntegerOps())) {
3678 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu8no_acc, RC: &ARM::tGPREvenRegClass, Op0);
3679 }
3680 return Register();
3681}
3682
3683Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(MVT RetVT, Register Op0) {
3684 if (RetVT.SimpleTy != MVT::i32)
3685 return Register();
3686 if ((Subtarget->hasMVEIntegerOps())) {
3687 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu16no_acc, RC: &ARM::tGPREvenRegClass, Op0);
3688 }
3689 return Register();
3690}
3691
3692Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(MVT RetVT, Register Op0) {
3693 if (RetVT.SimpleTy != MVT::i32)
3694 return Register();
3695 if ((Subtarget->hasMVEIntegerOps())) {
3696 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu32no_acc, RC: &ARM::tGPREvenRegClass, Op0);
3697 }
3698 return Register();
3699}
3700
3701Register fastEmit_ISD_VECREDUCE_ADD_r(MVT VT, MVT RetVT, Register Op0) {
3702 switch (VT.SimpleTy) {
3703 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(RetVT, Op0);
3704 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(RetVT, Op0);
3705 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(RetVT, Op0);
3706 default: return Register();
3707 }
3708}
3709
3710// FastEmit functions for ISD::ZERO_EXTEND.
3711
3712Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i8_r(MVT RetVT, Register Op0) {
3713 if (RetVT.SimpleTy != MVT::v8i16)
3714 return Register();
3715 if ((Subtarget->hasNEON())) {
3716 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv8i16, RC: &ARM::QPRRegClass, Op0);
3717 }
3718 return Register();
3719}
3720
3721Register fastEmit_ISD_ZERO_EXTEND_MVT_v4i16_r(MVT RetVT, Register Op0) {
3722 if (RetVT.SimpleTy != MVT::v4i32)
3723 return Register();
3724 if ((Subtarget->hasNEON())) {
3725 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv4i32, RC: &ARM::QPRRegClass, Op0);
3726 }
3727 return Register();
3728}
3729
3730Register fastEmit_ISD_ZERO_EXTEND_MVT_v2i32_r(MVT RetVT, Register Op0) {
3731 if (RetVT.SimpleTy != MVT::v2i64)
3732 return Register();
3733 if ((Subtarget->hasNEON())) {
3734 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv2i64, RC: &ARM::QPRRegClass, Op0);
3735 }
3736 return Register();
3737}
3738
3739Register fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3740 switch (VT.SimpleTy) {
3741 case MVT::v8i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i8_r(RetVT, Op0);
3742 case MVT::v4i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v4i16_r(RetVT, Op0);
3743 case MVT::v2i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v2i32_r(RetVT, Op0);
3744 default: return Register();
3745 }
3746}
3747
3748// Top-level FastEmit function.
3749
3750Register fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, Register Op0) override {
3751 switch (Opcode) {
3752 case ARMISD::VADDVs: return fastEmit_ARMISD_VADDVs_r(VT, RetVT, Op0);
3753 case ARMISD::VADDVu: return fastEmit_ARMISD_VADDVu_r(VT, RetVT, Op0);
3754 case ARMISD::VDUP: return fastEmit_ARMISD_VDUP_r(VT, RetVT, Op0);
3755 case ARMISD::VMOVhr: return fastEmit_ARMISD_VMOVhr_r(VT, RetVT, Op0);
3756 case ARMISD::VMOVrh: return fastEmit_ARMISD_VMOVrh_r(VT, RetVT, Op0);
3757 case ARMISD::VMOVsr: return fastEmit_ARMISD_VMOVsr_r(VT, RetVT, Op0);
3758 case ARMISD::VREV16: return fastEmit_ARMISD_VREV16_r(VT, RetVT, Op0);
3759 case ARMISD::VREV32: return fastEmit_ARMISD_VREV32_r(VT, RetVT, Op0);
3760 case ARMISD::VREV64: return fastEmit_ARMISD_VREV64_r(VT, RetVT, Op0);
3761 case ARMISD::WLSSETUP: return fastEmit_ARMISD_WLSSETUP_r(VT, RetVT, Op0);
3762 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
3763 case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0);
3764 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
3765 case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0);
3766 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
3767 case ISD::CTLS: return fastEmit_ISD_CTLS_r(VT, RetVT, Op0);
3768 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
3769 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
3770 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
3771 case ISD::FCEIL: return fastEmit_ISD_FCEIL_r(VT, RetVT, Op0);
3772 case ISD::FFLOOR: return fastEmit_ISD_FFLOOR_r(VT, RetVT, Op0);
3773 case ISD::FNEARBYINT: return fastEmit_ISD_FNEARBYINT_r(VT, RetVT, Op0);
3774 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
3775 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
3776 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
3777 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
3778 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
3779 case ISD::FRINT: return fastEmit_ISD_FRINT_r(VT, RetVT, Op0);
3780 case ISD::FROUND: return fastEmit_ISD_FROUND_r(VT, RetVT, Op0);
3781 case ISD::FROUNDEVEN: return fastEmit_ISD_FROUNDEVEN_r(VT, RetVT, Op0);
3782 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
3783 case ISD::FTRUNC: return fastEmit_ISD_FTRUNC_r(VT, RetVT, Op0);
3784 case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0);
3785 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
3786 case ISD::STRICT_FCEIL: return fastEmit_ISD_STRICT_FCEIL_r(VT, RetVT, Op0);
3787 case ISD::STRICT_FFLOOR: return fastEmit_ISD_STRICT_FFLOOR_r(VT, RetVT, Op0);
3788 case ISD::STRICT_FNEARBYINT: return fastEmit_ISD_STRICT_FNEARBYINT_r(VT, RetVT, Op0);
3789 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0);
3790 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0);
3791 case ISD::STRICT_FRINT: return fastEmit_ISD_STRICT_FRINT_r(VT, RetVT, Op0);
3792 case ISD::STRICT_FROUND: return fastEmit_ISD_STRICT_FROUND_r(VT, RetVT, Op0);
3793 case ISD::STRICT_FROUNDEVEN: return fastEmit_ISD_STRICT_FROUNDEVEN_r(VT, RetVT, Op0);
3794 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0);
3795 case ISD::STRICT_FTRUNC: return fastEmit_ISD_STRICT_FTRUNC_r(VT, RetVT, Op0);
3796 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
3797 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
3798 case ISD::VECREDUCE_ADD: return fastEmit_ISD_VECREDUCE_ADD_r(VT, RetVT, Op0);
3799 case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0);
3800 default: return Register();
3801 }
3802}
3803
3804// FastEmit functions for ARMISD::QADD16b.
3805
3806Register fastEmit_ARMISD_QADD16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3807 if (RetVT.SimpleTy != MVT::i32)
3808 return Register();
3809 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3810 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QADD16, RC: &ARM::rGPRRegClass, Op0, Op1);
3811 }
3812 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3813 return fastEmitInst_rr(MachineInstOpcode: ARM::QADD16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3814 }
3815 return Register();
3816}
3817
3818Register fastEmit_ARMISD_QADD16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3819 switch (VT.SimpleTy) {
3820 case MVT::i32: return fastEmit_ARMISD_QADD16b_MVT_i32_rr(RetVT, Op0, Op1);
3821 default: return Register();
3822 }
3823}
3824
3825// FastEmit functions for ARMISD::QADD8b.
3826
3827Register fastEmit_ARMISD_QADD8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3828 if (RetVT.SimpleTy != MVT::i32)
3829 return Register();
3830 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3831 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QADD8, RC: &ARM::rGPRRegClass, Op0, Op1);
3832 }
3833 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3834 return fastEmitInst_rr(MachineInstOpcode: ARM::QADD8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3835 }
3836 return Register();
3837}
3838
3839Register fastEmit_ARMISD_QADD8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3840 switch (VT.SimpleTy) {
3841 case MVT::i32: return fastEmit_ARMISD_QADD8b_MVT_i32_rr(RetVT, Op0, Op1);
3842 default: return Register();
3843 }
3844}
3845
3846// FastEmit functions for ARMISD::QSUB16b.
3847
3848Register fastEmit_ARMISD_QSUB16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3849 if (RetVT.SimpleTy != MVT::i32)
3850 return Register();
3851 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3852 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QSUB16, RC: &ARM::rGPRRegClass, Op0, Op1);
3853 }
3854 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3855 return fastEmitInst_rr(MachineInstOpcode: ARM::QSUB16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3856 }
3857 return Register();
3858}
3859
3860Register fastEmit_ARMISD_QSUB16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3861 switch (VT.SimpleTy) {
3862 case MVT::i32: return fastEmit_ARMISD_QSUB16b_MVT_i32_rr(RetVT, Op0, Op1);
3863 default: return Register();
3864 }
3865}
3866
3867// FastEmit functions for ARMISD::QSUB8b.
3868
3869Register fastEmit_ARMISD_QSUB8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3870 if (RetVT.SimpleTy != MVT::i32)
3871 return Register();
3872 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3873 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QSUB8, RC: &ARM::rGPRRegClass, Op0, Op1);
3874 }
3875 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3876 return fastEmitInst_rr(MachineInstOpcode: ARM::QSUB8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3877 }
3878 return Register();
3879}
3880
3881Register fastEmit_ARMISD_QSUB8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3882 switch (VT.SimpleTy) {
3883 case MVT::i32: return fastEmit_ARMISD_QSUB8b_MVT_i32_rr(RetVT, Op0, Op1);
3884 default: return Register();
3885 }
3886}
3887
3888// FastEmit functions for ARMISD::SMULWB.
3889
3890Register fastEmit_ARMISD_SMULWB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3891 if (RetVT.SimpleTy != MVT::i32)
3892 return Register();
3893 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3894 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SMULWB, RC: &ARM::rGPRRegClass, Op0, Op1);
3895 }
3896 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
3897 return fastEmitInst_rr(MachineInstOpcode: ARM::SMULWB, RC: &ARM::GPRRegClass, Op0, Op1);
3898 }
3899 return Register();
3900}
3901
3902Register fastEmit_ARMISD_SMULWB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3903 switch (VT.SimpleTy) {
3904 case MVT::i32: return fastEmit_ARMISD_SMULWB_MVT_i32_rr(RetVT, Op0, Op1);
3905 default: return Register();
3906 }
3907}
3908
3909// FastEmit functions for ARMISD::SMULWT.
3910
3911Register fastEmit_ARMISD_SMULWT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3912 if (RetVT.SimpleTy != MVT::i32)
3913 return Register();
3914 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3915 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SMULWT, RC: &ARM::rGPRRegClass, Op0, Op1);
3916 }
3917 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
3918 return fastEmitInst_rr(MachineInstOpcode: ARM::SMULWT, RC: &ARM::GPRRegClass, Op0, Op1);
3919 }
3920 return Register();
3921}
3922
3923Register fastEmit_ARMISD_SMULWT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3924 switch (VT.SimpleTy) {
3925 case MVT::i32: return fastEmit_ARMISD_SMULWT_MVT_i32_rr(RetVT, Op0, Op1);
3926 default: return Register();
3927 }
3928}
3929
3930// FastEmit functions for ARMISD::UQADD16b.
3931
3932Register fastEmit_ARMISD_UQADD16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3933 if (RetVT.SimpleTy != MVT::i32)
3934 return Register();
3935 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3936 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQADD16, RC: &ARM::rGPRRegClass, Op0, Op1);
3937 }
3938 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3939 return fastEmitInst_rr(MachineInstOpcode: ARM::UQADD16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3940 }
3941 return Register();
3942}
3943
3944Register fastEmit_ARMISD_UQADD16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3945 switch (VT.SimpleTy) {
3946 case MVT::i32: return fastEmit_ARMISD_UQADD16b_MVT_i32_rr(RetVT, Op0, Op1);
3947 default: return Register();
3948 }
3949}
3950
3951// FastEmit functions for ARMISD::UQADD8b.
3952
3953Register fastEmit_ARMISD_UQADD8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3954 if (RetVT.SimpleTy != MVT::i32)
3955 return Register();
3956 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3957 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQADD8, RC: &ARM::rGPRRegClass, Op0, Op1);
3958 }
3959 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3960 return fastEmitInst_rr(MachineInstOpcode: ARM::UQADD8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3961 }
3962 return Register();
3963}
3964
3965Register fastEmit_ARMISD_UQADD8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3966 switch (VT.SimpleTy) {
3967 case MVT::i32: return fastEmit_ARMISD_UQADD8b_MVT_i32_rr(RetVT, Op0, Op1);
3968 default: return Register();
3969 }
3970}
3971
3972// FastEmit functions for ARMISD::UQSUB16b.
3973
3974Register fastEmit_ARMISD_UQSUB16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3975 if (RetVT.SimpleTy != MVT::i32)
3976 return Register();
3977 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3978 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQSUB16, RC: &ARM::rGPRRegClass, Op0, Op1);
3979 }
3980 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3981 return fastEmitInst_rr(MachineInstOpcode: ARM::UQSUB16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3982 }
3983 return Register();
3984}
3985
3986Register fastEmit_ARMISD_UQSUB16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3987 switch (VT.SimpleTy) {
3988 case MVT::i32: return fastEmit_ARMISD_UQSUB16b_MVT_i32_rr(RetVT, Op0, Op1);
3989 default: return Register();
3990 }
3991}
3992
3993// FastEmit functions for ARMISD::UQSUB8b.
3994
3995Register fastEmit_ARMISD_UQSUB8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3996 if (RetVT.SimpleTy != MVT::i32)
3997 return Register();
3998 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3999 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQSUB8, RC: &ARM::rGPRRegClass, Op0, Op1);
4000 }
4001 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4002 return fastEmitInst_rr(MachineInstOpcode: ARM::UQSUB8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4003 }
4004 return Register();
4005}
4006
4007Register fastEmit_ARMISD_UQSUB8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4008 switch (VT.SimpleTy) {
4009 case MVT::i32: return fastEmit_ARMISD_UQSUB8b_MVT_i32_rr(RetVT, Op0, Op1);
4010 default: return Register();
4011 }
4012}
4013
4014// FastEmit functions for ARMISD::VMLAVs.
4015
4016Register fastEmit_ARMISD_VMLAVs_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4017 if (RetVT.SimpleTy != MVT::i32)
4018 return Register();
4019 if ((Subtarget->hasMVEIntegerOps())) {
4020 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVs8, RC: &ARM::tGPREvenRegClass, Op0, Op1);
4021 }
4022 return Register();
4023}
4024
4025Register fastEmit_ARMISD_VMLAVs_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4026 if (RetVT.SimpleTy != MVT::i32)
4027 return Register();
4028 if ((Subtarget->hasMVEIntegerOps())) {
4029 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVs16, RC: &ARM::tGPREvenRegClass, Op0, Op1);
4030 }
4031 return Register();
4032}
4033
4034Register fastEmit_ARMISD_VMLAVs_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4035 switch (VT.SimpleTy) {
4036 case MVT::v16i8: return fastEmit_ARMISD_VMLAVs_MVT_v16i8_rr(RetVT, Op0, Op1);
4037 case MVT::v8i16: return fastEmit_ARMISD_VMLAVs_MVT_v8i16_rr(RetVT, Op0, Op1);
4038 default: return Register();
4039 }
4040}
4041
4042// FastEmit functions for ARMISD::VMLAVu.
4043
4044Register fastEmit_ARMISD_VMLAVu_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4045 if (RetVT.SimpleTy != MVT::i32)
4046 return Register();
4047 if ((Subtarget->hasMVEIntegerOps())) {
4048 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVu8, RC: &ARM::tGPREvenRegClass, Op0, Op1);
4049 }
4050 return Register();
4051}
4052
4053Register fastEmit_ARMISD_VMLAVu_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4054 if (RetVT.SimpleTy != MVT::i32)
4055 return Register();
4056 if ((Subtarget->hasMVEIntegerOps())) {
4057 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVu16, RC: &ARM::tGPREvenRegClass, Op0, Op1);
4058 }
4059 return Register();
4060}
4061
4062Register fastEmit_ARMISD_VMLAVu_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4063 switch (VT.SimpleTy) {
4064 case MVT::v16i8: return fastEmit_ARMISD_VMLAVu_MVT_v16i8_rr(RetVT, Op0, Op1);
4065 case MVT::v8i16: return fastEmit_ARMISD_VMLAVu_MVT_v8i16_rr(RetVT, Op0, Op1);
4066 default: return Register();
4067 }
4068}
4069
4070// FastEmit functions for ARMISD::VMOVDRR.
4071
4072Register fastEmit_ARMISD_VMOVDRR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4073 if (RetVT.SimpleTy != MVT::f64)
4074 return Register();
4075 if ((Subtarget->hasFPRegs())) {
4076 return fastEmitInst_rr(MachineInstOpcode: ARM::VMOVDRR, RC: &ARM::DPRRegClass, Op0, Op1);
4077 }
4078 return Register();
4079}
4080
4081Register fastEmit_ARMISD_VMOVDRR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4082 switch (VT.SimpleTy) {
4083 case MVT::i32: return fastEmit_ARMISD_VMOVDRR_MVT_i32_rr(RetVT, Op0, Op1);
4084 default: return Register();
4085 }
4086}
4087
4088// FastEmit functions for ARMISD::VMULLs.
4089
4090Register fastEmit_ARMISD_VMULLs_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4091 if (RetVT.SimpleTy != MVT::v8i16)
4092 return Register();
4093 if ((Subtarget->hasNEON())) {
4094 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4095 }
4096 return Register();
4097}
4098
4099Register fastEmit_ARMISD_VMULLs_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4100 if (RetVT.SimpleTy != MVT::v4i32)
4101 return Register();
4102 if ((Subtarget->hasNEON())) {
4103 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4104 }
4105 return Register();
4106}
4107
4108Register fastEmit_ARMISD_VMULLs_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4109 if (RetVT.SimpleTy != MVT::v2i64)
4110 return Register();
4111 if ((Subtarget->hasNEON())) {
4112 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4113 }
4114 return Register();
4115}
4116
4117Register fastEmit_ARMISD_VMULLs_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4118 if (RetVT.SimpleTy != MVT::v2i64)
4119 return Register();
4120 if ((Subtarget->hasMVEIntegerOps())) {
4121 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULLBs32, RC: &ARM::MQPRRegClass, Op0, Op1);
4122 }
4123 return Register();
4124}
4125
4126Register fastEmit_ARMISD_VMULLs_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4127 switch (VT.SimpleTy) {
4128 case MVT::v8i8: return fastEmit_ARMISD_VMULLs_MVT_v8i8_rr(RetVT, Op0, Op1);
4129 case MVT::v4i16: return fastEmit_ARMISD_VMULLs_MVT_v4i16_rr(RetVT, Op0, Op1);
4130 case MVT::v2i32: return fastEmit_ARMISD_VMULLs_MVT_v2i32_rr(RetVT, Op0, Op1);
4131 case MVT::v4i32: return fastEmit_ARMISD_VMULLs_MVT_v4i32_rr(RetVT, Op0, Op1);
4132 default: return Register();
4133 }
4134}
4135
4136// FastEmit functions for ARMISD::VMULLu.
4137
4138Register fastEmit_ARMISD_VMULLu_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4139 if (RetVT.SimpleTy != MVT::v8i16)
4140 return Register();
4141 if ((Subtarget->hasNEON())) {
4142 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4143 }
4144 return Register();
4145}
4146
4147Register fastEmit_ARMISD_VMULLu_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4148 if (RetVT.SimpleTy != MVT::v4i32)
4149 return Register();
4150 if ((Subtarget->hasNEON())) {
4151 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4152 }
4153 return Register();
4154}
4155
4156Register fastEmit_ARMISD_VMULLu_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4157 if (RetVT.SimpleTy != MVT::v2i64)
4158 return Register();
4159 if ((Subtarget->hasNEON())) {
4160 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4161 }
4162 return Register();
4163}
4164
4165Register fastEmit_ARMISD_VMULLu_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4166 if (RetVT.SimpleTy != MVT::v2i64)
4167 return Register();
4168 if ((Subtarget->hasMVEIntegerOps())) {
4169 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULLBu32, RC: &ARM::MQPRRegClass, Op0, Op1);
4170 }
4171 return Register();
4172}
4173
4174Register fastEmit_ARMISD_VMULLu_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4175 switch (VT.SimpleTy) {
4176 case MVT::v8i8: return fastEmit_ARMISD_VMULLu_MVT_v8i8_rr(RetVT, Op0, Op1);
4177 case MVT::v4i16: return fastEmit_ARMISD_VMULLu_MVT_v4i16_rr(RetVT, Op0, Op1);
4178 case MVT::v2i32: return fastEmit_ARMISD_VMULLu_MVT_v2i32_rr(RetVT, Op0, Op1);
4179 case MVT::v4i32: return fastEmit_ARMISD_VMULLu_MVT_v4i32_rr(RetVT, Op0, Op1);
4180 default: return Register();
4181 }
4182}
4183
4184// FastEmit functions for ARMISD::VQDMULH.
4185
4186Register fastEmit_ARMISD_VQDMULH_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4187 if (RetVT.SimpleTy != MVT::v16i8)
4188 return Register();
4189 if ((Subtarget->hasMVEIntegerOps())) {
4190 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQDMULHi8, RC: &ARM::MQPRRegClass, Op0, Op1);
4191 }
4192 return Register();
4193}
4194
4195Register fastEmit_ARMISD_VQDMULH_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4196 if (RetVT.SimpleTy != MVT::v8i16)
4197 return Register();
4198 if ((Subtarget->hasMVEIntegerOps())) {
4199 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQDMULHi16, RC: &ARM::MQPRRegClass, Op0, Op1);
4200 }
4201 return Register();
4202}
4203
4204Register fastEmit_ARMISD_VQDMULH_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4205 if (RetVT.SimpleTy != MVT::v4i32)
4206 return Register();
4207 if ((Subtarget->hasMVEIntegerOps())) {
4208 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQDMULHi32, RC: &ARM::MQPRRegClass, Op0, Op1);
4209 }
4210 return Register();
4211}
4212
4213Register fastEmit_ARMISD_VQDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4214 switch (VT.SimpleTy) {
4215 case MVT::v16i8: return fastEmit_ARMISD_VQDMULH_MVT_v16i8_rr(RetVT, Op0, Op1);
4216 case MVT::v8i16: return fastEmit_ARMISD_VQDMULH_MVT_v8i16_rr(RetVT, Op0, Op1);
4217 case MVT::v4i32: return fastEmit_ARMISD_VQDMULH_MVT_v4i32_rr(RetVT, Op0, Op1);
4218 default: return Register();
4219 }
4220}
4221
4222// FastEmit functions for ARMISD::VSHLs.
4223
4224Register fastEmit_ARMISD_VSHLs_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4225 if (RetVT.SimpleTy != MVT::v8i8)
4226 return Register();
4227 if ((Subtarget->hasNEON())) {
4228 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4229 }
4230 return Register();
4231}
4232
4233Register fastEmit_ARMISD_VSHLs_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4234 if (RetVT.SimpleTy != MVT::v16i8)
4235 return Register();
4236 if ((Subtarget->hasMVEIntegerOps())) {
4237 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecs8, RC: &ARM::MQPRRegClass, Op0, Op1);
4238 }
4239 if ((Subtarget->hasNEON())) {
4240 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4241 }
4242 return Register();
4243}
4244
4245Register fastEmit_ARMISD_VSHLs_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4246 if (RetVT.SimpleTy != MVT::v4i16)
4247 return Register();
4248 if ((Subtarget->hasNEON())) {
4249 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4250 }
4251 return Register();
4252}
4253
4254Register fastEmit_ARMISD_VSHLs_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4255 if (RetVT.SimpleTy != MVT::v8i16)
4256 return Register();
4257 if ((Subtarget->hasMVEIntegerOps())) {
4258 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecs16, RC: &ARM::MQPRRegClass, Op0, Op1);
4259 }
4260 if ((Subtarget->hasNEON())) {
4261 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4262 }
4263 return Register();
4264}
4265
4266Register fastEmit_ARMISD_VSHLs_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4267 if (RetVT.SimpleTy != MVT::v2i32)
4268 return Register();
4269 if ((Subtarget->hasNEON())) {
4270 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4271 }
4272 return Register();
4273}
4274
4275Register fastEmit_ARMISD_VSHLs_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4276 if (RetVT.SimpleTy != MVT::v4i32)
4277 return Register();
4278 if ((Subtarget->hasMVEIntegerOps())) {
4279 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecs32, RC: &ARM::MQPRRegClass, Op0, Op1);
4280 }
4281 if ((Subtarget->hasNEON())) {
4282 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4283 }
4284 return Register();
4285}
4286
4287Register fastEmit_ARMISD_VSHLs_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
4288 if (RetVT.SimpleTy != MVT::v1i64)
4289 return Register();
4290 if ((Subtarget->hasNEON())) {
4291 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
4292 }
4293 return Register();
4294}
4295
4296Register fastEmit_ARMISD_VSHLs_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
4297 if (RetVT.SimpleTy != MVT::v2i64)
4298 return Register();
4299 if ((Subtarget->hasNEON())) {
4300 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4301 }
4302 return Register();
4303}
4304
4305Register fastEmit_ARMISD_VSHLs_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4306 switch (VT.SimpleTy) {
4307 case MVT::v8i8: return fastEmit_ARMISD_VSHLs_MVT_v8i8_rr(RetVT, Op0, Op1);
4308 case MVT::v16i8: return fastEmit_ARMISD_VSHLs_MVT_v16i8_rr(RetVT, Op0, Op1);
4309 case MVT::v4i16: return fastEmit_ARMISD_VSHLs_MVT_v4i16_rr(RetVT, Op0, Op1);
4310 case MVT::v8i16: return fastEmit_ARMISD_VSHLs_MVT_v8i16_rr(RetVT, Op0, Op1);
4311 case MVT::v2i32: return fastEmit_ARMISD_VSHLs_MVT_v2i32_rr(RetVT, Op0, Op1);
4312 case MVT::v4i32: return fastEmit_ARMISD_VSHLs_MVT_v4i32_rr(RetVT, Op0, Op1);
4313 case MVT::v1i64: return fastEmit_ARMISD_VSHLs_MVT_v1i64_rr(RetVT, Op0, Op1);
4314 case MVT::v2i64: return fastEmit_ARMISD_VSHLs_MVT_v2i64_rr(RetVT, Op0, Op1);
4315 default: return Register();
4316 }
4317}
4318
4319// FastEmit functions for ARMISD::VSHLu.
4320
4321Register fastEmit_ARMISD_VSHLu_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4322 if (RetVT.SimpleTy != MVT::v8i8)
4323 return Register();
4324 if ((Subtarget->hasNEON())) {
4325 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4326 }
4327 return Register();
4328}
4329
4330Register fastEmit_ARMISD_VSHLu_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4331 if (RetVT.SimpleTy != MVT::v16i8)
4332 return Register();
4333 if ((Subtarget->hasMVEIntegerOps())) {
4334 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecu8, RC: &ARM::MQPRRegClass, Op0, Op1);
4335 }
4336 if ((Subtarget->hasNEON())) {
4337 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4338 }
4339 return Register();
4340}
4341
4342Register fastEmit_ARMISD_VSHLu_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4343 if (RetVT.SimpleTy != MVT::v4i16)
4344 return Register();
4345 if ((Subtarget->hasNEON())) {
4346 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4347 }
4348 return Register();
4349}
4350
4351Register fastEmit_ARMISD_VSHLu_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4352 if (RetVT.SimpleTy != MVT::v8i16)
4353 return Register();
4354 if ((Subtarget->hasMVEIntegerOps())) {
4355 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecu16, RC: &ARM::MQPRRegClass, Op0, Op1);
4356 }
4357 if ((Subtarget->hasNEON())) {
4358 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4359 }
4360 return Register();
4361}
4362
4363Register fastEmit_ARMISD_VSHLu_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4364 if (RetVT.SimpleTy != MVT::v2i32)
4365 return Register();
4366 if ((Subtarget->hasNEON())) {
4367 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4368 }
4369 return Register();
4370}
4371
4372Register fastEmit_ARMISD_VSHLu_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4373 if (RetVT.SimpleTy != MVT::v4i32)
4374 return Register();
4375 if ((Subtarget->hasMVEIntegerOps())) {
4376 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecu32, RC: &ARM::MQPRRegClass, Op0, Op1);
4377 }
4378 if ((Subtarget->hasNEON())) {
4379 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4380 }
4381 return Register();
4382}
4383
4384Register fastEmit_ARMISD_VSHLu_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
4385 if (RetVT.SimpleTy != MVT::v1i64)
4386 return Register();
4387 if ((Subtarget->hasNEON())) {
4388 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
4389 }
4390 return Register();
4391}
4392
4393Register fastEmit_ARMISD_VSHLu_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
4394 if (RetVT.SimpleTy != MVT::v2i64)
4395 return Register();
4396 if ((Subtarget->hasNEON())) {
4397 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4398 }
4399 return Register();
4400}
4401
4402Register fastEmit_ARMISD_VSHLu_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4403 switch (VT.SimpleTy) {
4404 case MVT::v8i8: return fastEmit_ARMISD_VSHLu_MVT_v8i8_rr(RetVT, Op0, Op1);
4405 case MVT::v16i8: return fastEmit_ARMISD_VSHLu_MVT_v16i8_rr(RetVT, Op0, Op1);
4406 case MVT::v4i16: return fastEmit_ARMISD_VSHLu_MVT_v4i16_rr(RetVT, Op0, Op1);
4407 case MVT::v8i16: return fastEmit_ARMISD_VSHLu_MVT_v8i16_rr(RetVT, Op0, Op1);
4408 case MVT::v2i32: return fastEmit_ARMISD_VSHLu_MVT_v2i32_rr(RetVT, Op0, Op1);
4409 case MVT::v4i32: return fastEmit_ARMISD_VSHLu_MVT_v4i32_rr(RetVT, Op0, Op1);
4410 case MVT::v1i64: return fastEmit_ARMISD_VSHLu_MVT_v1i64_rr(RetVT, Op0, Op1);
4411 case MVT::v2i64: return fastEmit_ARMISD_VSHLu_MVT_v2i64_rr(RetVT, Op0, Op1);
4412 default: return Register();
4413 }
4414}
4415
4416// FastEmit functions for ARMISD::VTBL1.
4417
4418Register fastEmit_ARMISD_VTBL1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4419 if (RetVT.SimpleTy != MVT::v8i8)
4420 return Register();
4421 if ((Subtarget->hasNEON())) {
4422 return fastEmitInst_rr(MachineInstOpcode: ARM::VTBL1, RC: &ARM::DPRRegClass, Op0, Op1);
4423 }
4424 return Register();
4425}
4426
4427Register fastEmit_ARMISD_VTBL1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4428 switch (VT.SimpleTy) {
4429 case MVT::v8i8: return fastEmit_ARMISD_VTBL1_MVT_v8i8_rr(RetVT, Op0, Op1);
4430 default: return Register();
4431 }
4432}
4433
4434// FastEmit functions for ARMISD::VTST.
4435
4436Register fastEmit_ARMISD_VTST_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4437 if (RetVT.SimpleTy != MVT::v8i8)
4438 return Register();
4439 if ((Subtarget->hasNEON())) {
4440 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4441 }
4442 return Register();
4443}
4444
4445Register fastEmit_ARMISD_VTST_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4446 if (RetVT.SimpleTy != MVT::v16i8)
4447 return Register();
4448 if ((Subtarget->hasNEON())) {
4449 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4450 }
4451 return Register();
4452}
4453
4454Register fastEmit_ARMISD_VTST_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4455 if (RetVT.SimpleTy != MVT::v4i16)
4456 return Register();
4457 if ((Subtarget->hasNEON())) {
4458 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4459 }
4460 return Register();
4461}
4462
4463Register fastEmit_ARMISD_VTST_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4464 if (RetVT.SimpleTy != MVT::v8i16)
4465 return Register();
4466 if ((Subtarget->hasNEON())) {
4467 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4468 }
4469 return Register();
4470}
4471
4472Register fastEmit_ARMISD_VTST_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4473 if (RetVT.SimpleTy != MVT::v2i32)
4474 return Register();
4475 if ((Subtarget->hasNEON())) {
4476 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4477 }
4478 return Register();
4479}
4480
4481Register fastEmit_ARMISD_VTST_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4482 if (RetVT.SimpleTy != MVT::v4i32)
4483 return Register();
4484 if ((Subtarget->hasNEON())) {
4485 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4486 }
4487 return Register();
4488}
4489
4490Register fastEmit_ARMISD_VTST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4491 switch (VT.SimpleTy) {
4492 case MVT::v8i8: return fastEmit_ARMISD_VTST_MVT_v8i8_rr(RetVT, Op0, Op1);
4493 case MVT::v16i8: return fastEmit_ARMISD_VTST_MVT_v16i8_rr(RetVT, Op0, Op1);
4494 case MVT::v4i16: return fastEmit_ARMISD_VTST_MVT_v4i16_rr(RetVT, Op0, Op1);
4495 case MVT::v8i16: return fastEmit_ARMISD_VTST_MVT_v8i16_rr(RetVT, Op0, Op1);
4496 case MVT::v2i32: return fastEmit_ARMISD_VTST_MVT_v2i32_rr(RetVT, Op0, Op1);
4497 case MVT::v4i32: return fastEmit_ARMISD_VTST_MVT_v4i32_rr(RetVT, Op0, Op1);
4498 default: return Register();
4499 }
4500}
4501
4502// FastEmit functions for ISD::ABDS.
4503
4504Register fastEmit_ISD_ABDS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4505 if (RetVT.SimpleTy != MVT::v8i8)
4506 return Register();
4507 if ((Subtarget->hasNEON())) {
4508 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4509 }
4510 return Register();
4511}
4512
4513Register fastEmit_ISD_ABDS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4514 if (RetVT.SimpleTy != MVT::v16i8)
4515 return Register();
4516 if ((Subtarget->hasMVEIntegerOps())) {
4517 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
4518 }
4519 if ((Subtarget->hasNEON())) {
4520 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4521 }
4522 return Register();
4523}
4524
4525Register fastEmit_ISD_ABDS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4526 if (RetVT.SimpleTy != MVT::v4i16)
4527 return Register();
4528 if ((Subtarget->hasNEON())) {
4529 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4530 }
4531 return Register();
4532}
4533
4534Register fastEmit_ISD_ABDS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4535 if (RetVT.SimpleTy != MVT::v8i16)
4536 return Register();
4537 if ((Subtarget->hasMVEIntegerOps())) {
4538 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
4539 }
4540 if ((Subtarget->hasNEON())) {
4541 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4542 }
4543 return Register();
4544}
4545
4546Register fastEmit_ISD_ABDS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4547 if (RetVT.SimpleTy != MVT::v2i32)
4548 return Register();
4549 if ((Subtarget->hasNEON())) {
4550 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4551 }
4552 return Register();
4553}
4554
4555Register fastEmit_ISD_ABDS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4556 if (RetVT.SimpleTy != MVT::v4i32)
4557 return Register();
4558 if ((Subtarget->hasMVEIntegerOps())) {
4559 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
4560 }
4561 if ((Subtarget->hasNEON())) {
4562 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4563 }
4564 return Register();
4565}
4566
4567Register fastEmit_ISD_ABDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4568 switch (VT.SimpleTy) {
4569 case MVT::v8i8: return fastEmit_ISD_ABDS_MVT_v8i8_rr(RetVT, Op0, Op1);
4570 case MVT::v16i8: return fastEmit_ISD_ABDS_MVT_v16i8_rr(RetVT, Op0, Op1);
4571 case MVT::v4i16: return fastEmit_ISD_ABDS_MVT_v4i16_rr(RetVT, Op0, Op1);
4572 case MVT::v8i16: return fastEmit_ISD_ABDS_MVT_v8i16_rr(RetVT, Op0, Op1);
4573 case MVT::v2i32: return fastEmit_ISD_ABDS_MVT_v2i32_rr(RetVT, Op0, Op1);
4574 case MVT::v4i32: return fastEmit_ISD_ABDS_MVT_v4i32_rr(RetVT, Op0, Op1);
4575 default: return Register();
4576 }
4577}
4578
4579// FastEmit functions for ISD::ABDU.
4580
4581Register fastEmit_ISD_ABDU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4582 if (RetVT.SimpleTy != MVT::v8i8)
4583 return Register();
4584 if ((Subtarget->hasNEON())) {
4585 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4586 }
4587 return Register();
4588}
4589
4590Register fastEmit_ISD_ABDU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4591 if (RetVT.SimpleTy != MVT::v16i8)
4592 return Register();
4593 if ((Subtarget->hasMVEIntegerOps())) {
4594 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
4595 }
4596 if ((Subtarget->hasNEON())) {
4597 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4598 }
4599 return Register();
4600}
4601
4602Register fastEmit_ISD_ABDU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4603 if (RetVT.SimpleTy != MVT::v4i16)
4604 return Register();
4605 if ((Subtarget->hasNEON())) {
4606 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4607 }
4608 return Register();
4609}
4610
4611Register fastEmit_ISD_ABDU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4612 if (RetVT.SimpleTy != MVT::v8i16)
4613 return Register();
4614 if ((Subtarget->hasMVEIntegerOps())) {
4615 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
4616 }
4617 if ((Subtarget->hasNEON())) {
4618 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4619 }
4620 return Register();
4621}
4622
4623Register fastEmit_ISD_ABDU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4624 if (RetVT.SimpleTy != MVT::v2i32)
4625 return Register();
4626 if ((Subtarget->hasNEON())) {
4627 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4628 }
4629 return Register();
4630}
4631
4632Register fastEmit_ISD_ABDU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4633 if (RetVT.SimpleTy != MVT::v4i32)
4634 return Register();
4635 if ((Subtarget->hasMVEIntegerOps())) {
4636 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
4637 }
4638 if ((Subtarget->hasNEON())) {
4639 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4640 }
4641 return Register();
4642}
4643
4644Register fastEmit_ISD_ABDU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4645 switch (VT.SimpleTy) {
4646 case MVT::v8i8: return fastEmit_ISD_ABDU_MVT_v8i8_rr(RetVT, Op0, Op1);
4647 case MVT::v16i8: return fastEmit_ISD_ABDU_MVT_v16i8_rr(RetVT, Op0, Op1);
4648 case MVT::v4i16: return fastEmit_ISD_ABDU_MVT_v4i16_rr(RetVT, Op0, Op1);
4649 case MVT::v8i16: return fastEmit_ISD_ABDU_MVT_v8i16_rr(RetVT, Op0, Op1);
4650 case MVT::v2i32: return fastEmit_ISD_ABDU_MVT_v2i32_rr(RetVT, Op0, Op1);
4651 case MVT::v4i32: return fastEmit_ISD_ABDU_MVT_v4i32_rr(RetVT, Op0, Op1);
4652 default: return Register();
4653 }
4654}
4655
4656// FastEmit functions for ISD::ADD.
4657
4658Register fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4659 if (RetVT.SimpleTy != MVT::i32)
4660 return Register();
4661 if ((Subtarget->isThumb2())) {
4662 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ADDrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4663 }
4664 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
4665 return fastEmitInst_rr(MachineInstOpcode: ARM::tADDrr, RC: &ARM::tGPRRegClass, Op0, Op1);
4666 }
4667 if ((!Subtarget->isThumb())) {
4668 return fastEmitInst_rr(MachineInstOpcode: ARM::ADDrr, RC: &ARM::GPRRegClass, Op0, Op1);
4669 }
4670 return Register();
4671}
4672
4673Register fastEmit_ISD_ADD_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4674 if (RetVT.SimpleTy != MVT::v8i8)
4675 return Register();
4676 if ((Subtarget->hasNEON())) {
4677 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4678 }
4679 return Register();
4680}
4681
4682Register fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4683 if (RetVT.SimpleTy != MVT::v16i8)
4684 return Register();
4685 if ((Subtarget->hasMVEIntegerOps())) {
4686 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDi8, RC: &ARM::MQPRRegClass, Op0, Op1);
4687 }
4688 if ((Subtarget->hasNEON())) {
4689 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4690 }
4691 return Register();
4692}
4693
4694Register fastEmit_ISD_ADD_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4695 if (RetVT.SimpleTy != MVT::v4i16)
4696 return Register();
4697 if ((Subtarget->hasNEON())) {
4698 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4699 }
4700 return Register();
4701}
4702
4703Register fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4704 if (RetVT.SimpleTy != MVT::v8i16)
4705 return Register();
4706 if ((Subtarget->hasMVEIntegerOps())) {
4707 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDi16, RC: &ARM::MQPRRegClass, Op0, Op1);
4708 }
4709 if ((Subtarget->hasNEON())) {
4710 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4711 }
4712 return Register();
4713}
4714
4715Register fastEmit_ISD_ADD_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4716 if (RetVT.SimpleTy != MVT::v2i32)
4717 return Register();
4718 if ((Subtarget->hasNEON())) {
4719 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4720 }
4721 return Register();
4722}
4723
4724Register fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4725 if (RetVT.SimpleTy != MVT::v4i32)
4726 return Register();
4727 if ((Subtarget->hasMVEIntegerOps())) {
4728 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDi32, RC: &ARM::MQPRRegClass, Op0, Op1);
4729 }
4730 if ((Subtarget->hasNEON())) {
4731 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4732 }
4733 return Register();
4734}
4735
4736Register fastEmit_ISD_ADD_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
4737 if (RetVT.SimpleTy != MVT::v1i64)
4738 return Register();
4739 if ((Subtarget->hasNEON())) {
4740 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
4741 }
4742 return Register();
4743}
4744
4745Register fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
4746 if (RetVT.SimpleTy != MVT::v2i64)
4747 return Register();
4748 if ((Subtarget->hasNEON())) {
4749 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4750 }
4751 return Register();
4752}
4753
4754Register fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4755 switch (VT.SimpleTy) {
4756 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
4757 case MVT::v8i8: return fastEmit_ISD_ADD_MVT_v8i8_rr(RetVT, Op0, Op1);
4758 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
4759 case MVT::v4i16: return fastEmit_ISD_ADD_MVT_v4i16_rr(RetVT, Op0, Op1);
4760 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
4761 case MVT::v2i32: return fastEmit_ISD_ADD_MVT_v2i32_rr(RetVT, Op0, Op1);
4762 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
4763 case MVT::v1i64: return fastEmit_ISD_ADD_MVT_v1i64_rr(RetVT, Op0, Op1);
4764 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
4765 default: return Register();
4766 }
4767}
4768
4769// FastEmit functions for ISD::AND.
4770
4771Register fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4772 if (RetVT.SimpleTy != MVT::i32)
4773 return Register();
4774 if ((Subtarget->isThumb2())) {
4775 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ANDrr, RC: &ARM::rGPRRegClass, Op0, Op1);
4776 }
4777 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
4778 return fastEmitInst_rr(MachineInstOpcode: ARM::tAND, RC: &ARM::tGPRRegClass, Op0, Op1);
4779 }
4780 if ((!Subtarget->isThumb())) {
4781 return fastEmitInst_rr(MachineInstOpcode: ARM::ANDrr, RC: &ARM::GPRRegClass, Op0, Op1);
4782 }
4783 return Register();
4784}
4785
4786Register fastEmit_ISD_AND_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4787 if (RetVT.SimpleTy != MVT::v8i8)
4788 return Register();
4789 if ((Subtarget->hasNEON())) {
4790 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
4791 }
4792 return Register();
4793}
4794
4795Register fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4796 if (RetVT.SimpleTy != MVT::v16i8)
4797 return Register();
4798 if ((Subtarget->hasMVEIntegerOps())) {
4799 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
4800 }
4801 if ((Subtarget->hasNEON())) {
4802 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
4803 }
4804 return Register();
4805}
4806
4807Register fastEmit_ISD_AND_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4808 if (RetVT.SimpleTy != MVT::v4i16)
4809 return Register();
4810 if ((Subtarget->hasNEON())) {
4811 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
4812 }
4813 return Register();
4814}
4815
4816Register fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4817 if (RetVT.SimpleTy != MVT::v8i16)
4818 return Register();
4819 if ((Subtarget->hasMVEIntegerOps())) {
4820 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
4821 }
4822 if ((Subtarget->hasNEON())) {
4823 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
4824 }
4825 return Register();
4826}
4827
4828Register fastEmit_ISD_AND_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4829 if (RetVT.SimpleTy != MVT::v2i32)
4830 return Register();
4831 if ((Subtarget->hasNEON())) {
4832 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
4833 }
4834 return Register();
4835}
4836
4837Register fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4838 if (RetVT.SimpleTy != MVT::v4i32)
4839 return Register();
4840 if ((Subtarget->hasMVEIntegerOps())) {
4841 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
4842 }
4843 if ((Subtarget->hasNEON())) {
4844 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
4845 }
4846 return Register();
4847}
4848
4849Register fastEmit_ISD_AND_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
4850 if (RetVT.SimpleTy != MVT::v1i64)
4851 return Register();
4852 if ((Subtarget->hasNEON())) {
4853 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
4854 }
4855 return Register();
4856}
4857
4858Register fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
4859 if (RetVT.SimpleTy != MVT::v2i64)
4860 return Register();
4861 if ((Subtarget->hasMVEIntegerOps())) {
4862 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
4863 }
4864 if ((Subtarget->hasNEON())) {
4865 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
4866 }
4867 return Register();
4868}
4869
4870Register fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4871 switch (VT.SimpleTy) {
4872 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
4873 case MVT::v8i8: return fastEmit_ISD_AND_MVT_v8i8_rr(RetVT, Op0, Op1);
4874 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
4875 case MVT::v4i16: return fastEmit_ISD_AND_MVT_v4i16_rr(RetVT, Op0, Op1);
4876 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
4877 case MVT::v2i32: return fastEmit_ISD_AND_MVT_v2i32_rr(RetVT, Op0, Op1);
4878 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
4879 case MVT::v1i64: return fastEmit_ISD_AND_MVT_v1i64_rr(RetVT, Op0, Op1);
4880 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
4881 default: return Register();
4882 }
4883}
4884
4885// FastEmit functions for ISD::AVGCEILS.
4886
4887Register fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4888 if (RetVT.SimpleTy != MVT::v16i8)
4889 return Register();
4890 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
4891}
4892
4893Register fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4894 if (RetVT.SimpleTy != MVT::v8i16)
4895 return Register();
4896 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
4897}
4898
4899Register fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4900 if (RetVT.SimpleTy != MVT::v4i32)
4901 return Register();
4902 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
4903}
4904
4905Register fastEmit_ISD_AVGCEILS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4906 switch (VT.SimpleTy) {
4907 case MVT::v16i8: return fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(RetVT, Op0, Op1);
4908 case MVT::v8i16: return fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(RetVT, Op0, Op1);
4909 case MVT::v4i32: return fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(RetVT, Op0, Op1);
4910 default: return Register();
4911 }
4912}
4913
4914// FastEmit functions for ISD::AVGCEILU.
4915
4916Register fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4917 if (RetVT.SimpleTy != MVT::v16i8)
4918 return Register();
4919 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
4920}
4921
4922Register fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4923 if (RetVT.SimpleTy != MVT::v8i16)
4924 return Register();
4925 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
4926}
4927
4928Register fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4929 if (RetVT.SimpleTy != MVT::v4i32)
4930 return Register();
4931 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
4932}
4933
4934Register fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4935 switch (VT.SimpleTy) {
4936 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
4937 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
4938 case MVT::v4i32: return fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(RetVT, Op0, Op1);
4939 default: return Register();
4940 }
4941}
4942
4943// FastEmit functions for ISD::AVGFLOORS.
4944
4945Register fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4946 if (RetVT.SimpleTy != MVT::v16i8)
4947 return Register();
4948 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
4949}
4950
4951Register fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4952 if (RetVT.SimpleTy != MVT::v8i16)
4953 return Register();
4954 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
4955}
4956
4957Register fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4958 if (RetVT.SimpleTy != MVT::v4i32)
4959 return Register();
4960 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
4961}
4962
4963Register fastEmit_ISD_AVGFLOORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4964 switch (VT.SimpleTy) {
4965 case MVT::v16i8: return fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(RetVT, Op0, Op1);
4966 case MVT::v8i16: return fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(RetVT, Op0, Op1);
4967 case MVT::v4i32: return fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(RetVT, Op0, Op1);
4968 default: return Register();
4969 }
4970}
4971
4972// FastEmit functions for ISD::AVGFLOORU.
4973
4974Register fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4975 if (RetVT.SimpleTy != MVT::v16i8)
4976 return Register();
4977 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
4978}
4979
4980Register fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4981 if (RetVT.SimpleTy != MVT::v8i16)
4982 return Register();
4983 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
4984}
4985
4986Register fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4987 if (RetVT.SimpleTy != MVT::v4i32)
4988 return Register();
4989 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
4990}
4991
4992Register fastEmit_ISD_AVGFLOORU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4993 switch (VT.SimpleTy) {
4994 case MVT::v16i8: return fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(RetVT, Op0, Op1);
4995 case MVT::v8i16: return fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(RetVT, Op0, Op1);
4996 case MVT::v4i32: return fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(RetVT, Op0, Op1);
4997 default: return Register();
4998 }
4999}
5000
5001// FastEmit functions for ISD::FADD.
5002
5003Register fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5004 if (RetVT.SimpleTy != MVT::f16)
5005 return Register();
5006 if ((Subtarget->hasFullFP16())) {
5007 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDH, RC: &ARM::HPRRegClass, Op0, Op1);
5008 }
5009 return Register();
5010}
5011
5012Register fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5013 if (RetVT.SimpleTy != MVT::f32)
5014 return Register();
5015 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
5016 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDS, RC: &ARM::SPRRegClass, Op0, Op1);
5017 }
5018 return Register();
5019}
5020
5021Register fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5022 if (RetVT.SimpleTy != MVT::f64)
5023 return Register();
5024 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
5025 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDD, RC: &ARM::DPRRegClass, Op0, Op1);
5026 }
5027 return Register();
5028}
5029
5030Register fastEmit_ISD_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5031 if (RetVT.SimpleTy != MVT::v4f16)
5032 return Register();
5033 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5034 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDhd, RC: &ARM::DPRRegClass, Op0, Op1);
5035 }
5036 return Register();
5037}
5038
5039Register fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5040 if (RetVT.SimpleTy != MVT::v8f16)
5041 return Register();
5042 if ((Subtarget->hasMVEFloatOps())) {
5043 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5044 }
5045 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5046 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDhq, RC: &ARM::QPRRegClass, Op0, Op1);
5047 }
5048 return Register();
5049}
5050
5051Register fastEmit_ISD_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5052 if (RetVT.SimpleTy != MVT::v2f32)
5053 return Register();
5054 if ((Subtarget->hasNEON())) {
5055 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDfd, RC: &ARM::DPRRegClass, Op0, Op1);
5056 }
5057 return Register();
5058}
5059
5060Register fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5061 if (RetVT.SimpleTy != MVT::v4f32)
5062 return Register();
5063 if ((Subtarget->hasMVEFloatOps())) {
5064 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5065 }
5066 if ((Subtarget->hasNEON())) {
5067 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDfq, RC: &ARM::QPRRegClass, Op0, Op1);
5068 }
5069 return Register();
5070}
5071
5072Register fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5073 switch (VT.SimpleTy) {
5074 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
5075 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
5076 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
5077 case MVT::v4f16: return fastEmit_ISD_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
5078 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
5079 case MVT::v2f32: return fastEmit_ISD_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
5080 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
5081 default: return Register();
5082 }
5083}
5084
5085// FastEmit functions for ISD::FDIV.
5086
5087Register fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5088 if (RetVT.SimpleTy != MVT::f16)
5089 return Register();
5090 if ((Subtarget->hasFullFP16())) {
5091 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVH, RC: &ARM::HPRRegClass, Op0, Op1);
5092 }
5093 return Register();
5094}
5095
5096Register fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5097 if (RetVT.SimpleTy != MVT::f32)
5098 return Register();
5099 if ((Subtarget->hasVFP2Base())) {
5100 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVS, RC: &ARM::SPRRegClass, Op0, Op1);
5101 }
5102 return Register();
5103}
5104
5105Register fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5106 if (RetVT.SimpleTy != MVT::f64)
5107 return Register();
5108 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
5109 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVD, RC: &ARM::DPRRegClass, Op0, Op1);
5110 }
5111 return Register();
5112}
5113
5114Register fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5115 switch (VT.SimpleTy) {
5116 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
5117 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
5118 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
5119 default: return Register();
5120 }
5121}
5122
5123// FastEmit functions for ISD::FMAXIMUM.
5124
5125Register fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5126 if (RetVT.SimpleTy != MVT::v4f16)
5127 return Register();
5128 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5129 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXhd, RC: &ARM::DPRRegClass, Op0, Op1);
5130 }
5131 return Register();
5132}
5133
5134Register fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5135 if (RetVT.SimpleTy != MVT::v8f16)
5136 return Register();
5137 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5138 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXhq, RC: &ARM::QPRRegClass, Op0, Op1);
5139 }
5140 return Register();
5141}
5142
5143Register fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5144 if (RetVT.SimpleTy != MVT::v2f32)
5145 return Register();
5146 if ((Subtarget->hasNEON())) {
5147 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXfd, RC: &ARM::DPRRegClass, Op0, Op1);
5148 }
5149 return Register();
5150}
5151
5152Register fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5153 if (RetVT.SimpleTy != MVT::v4f32)
5154 return Register();
5155 if ((Subtarget->hasNEON())) {
5156 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXfq, RC: &ARM::QPRRegClass, Op0, Op1);
5157 }
5158 return Register();
5159}
5160
5161Register fastEmit_ISD_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5162 switch (VT.SimpleTy) {
5163 case MVT::v4f16: return fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
5164 case MVT::v8f16: return fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
5165 case MVT::v2f32: return fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
5166 case MVT::v4f32: return fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
5167 default: return Register();
5168 }
5169}
5170
5171// FastEmit functions for ISD::FMAXNUM.
5172
5173Register fastEmit_ISD_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5174 if (RetVT.SimpleTy != MVT::f16)
5175 return Register();
5176 if ((Subtarget->hasFullFP16())) {
5177 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMH, RC: &ARM::HPRRegClass, Op0, Op1);
5178 }
5179 return Register();
5180}
5181
5182Register fastEmit_ISD_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5183 if (RetVT.SimpleTy != MVT::f32)
5184 return Register();
5185 if ((Subtarget->hasFPARMv8Base())) {
5186 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMS, RC: &ARM::SPRRegClass, Op0, Op1);
5187 }
5188 return Register();
5189}
5190
5191Register fastEmit_ISD_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5192 if (RetVT.SimpleTy != MVT::f64)
5193 return Register();
5194 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
5195 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMD, RC: &ARM::DPRRegClass, Op0, Op1);
5196 }
5197 return Register();
5198}
5199
5200Register fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5201 if (RetVT.SimpleTy != MVT::v4f16)
5202 return Register();
5203 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5204 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNDh, RC: &ARM::DPRRegClass, Op0, Op1);
5205 }
5206 return Register();
5207}
5208
5209Register fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5210 if (RetVT.SimpleTy != MVT::v8f16)
5211 return Register();
5212 if ((Subtarget->hasMVEFloatOps())) {
5213 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXNMf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5214 }
5215 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5216 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNQh, RC: &ARM::QPRRegClass, Op0, Op1);
5217 }
5218 return Register();
5219}
5220
5221Register fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5222 if (RetVT.SimpleTy != MVT::v2f32)
5223 return Register();
5224 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
5225 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNDf, RC: &ARM::DPRRegClass, Op0, Op1);
5226 }
5227 return Register();
5228}
5229
5230Register fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5231 if (RetVT.SimpleTy != MVT::v4f32)
5232 return Register();
5233 if ((Subtarget->hasMVEFloatOps())) {
5234 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXNMf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5235 }
5236 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
5237 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNQf, RC: &ARM::QPRRegClass, Op0, Op1);
5238 }
5239 return Register();
5240}
5241
5242Register fastEmit_ISD_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5243 switch (VT.SimpleTy) {
5244 case MVT::f16: return fastEmit_ISD_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
5245 case MVT::f32: return fastEmit_ISD_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
5246 case MVT::f64: return fastEmit_ISD_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
5247 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
5248 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
5249 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
5250 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
5251 default: return Register();
5252 }
5253}
5254
5255// FastEmit functions for ISD::FMINIMUM.
5256
5257Register fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5258 if (RetVT.SimpleTy != MVT::v4f16)
5259 return Register();
5260 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5261 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINhd, RC: &ARM::DPRRegClass, Op0, Op1);
5262 }
5263 return Register();
5264}
5265
5266Register fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5267 if (RetVT.SimpleTy != MVT::v8f16)
5268 return Register();
5269 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5270 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINhq, RC: &ARM::QPRRegClass, Op0, Op1);
5271 }
5272 return Register();
5273}
5274
5275Register fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5276 if (RetVT.SimpleTy != MVT::v2f32)
5277 return Register();
5278 if ((Subtarget->hasNEON())) {
5279 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINfd, RC: &ARM::DPRRegClass, Op0, Op1);
5280 }
5281 return Register();
5282}
5283
5284Register fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5285 if (RetVT.SimpleTy != MVT::v4f32)
5286 return Register();
5287 if ((Subtarget->hasNEON())) {
5288 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINfq, RC: &ARM::QPRRegClass, Op0, Op1);
5289 }
5290 return Register();
5291}
5292
5293Register fastEmit_ISD_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5294 switch (VT.SimpleTy) {
5295 case MVT::v4f16: return fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
5296 case MVT::v8f16: return fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
5297 case MVT::v2f32: return fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
5298 case MVT::v4f32: return fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
5299 default: return Register();
5300 }
5301}
5302
5303// FastEmit functions for ISD::FMINNUM.
5304
5305Register fastEmit_ISD_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5306 if (RetVT.SimpleTy != MVT::f16)
5307 return Register();
5308 if ((Subtarget->hasFullFP16())) {
5309 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMH, RC: &ARM::HPRRegClass, Op0, Op1);
5310 }
5311 return Register();
5312}
5313
5314Register fastEmit_ISD_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5315 if (RetVT.SimpleTy != MVT::f32)
5316 return Register();
5317 if ((Subtarget->hasFPARMv8Base())) {
5318 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMS, RC: &ARM::SPRRegClass, Op0, Op1);
5319 }
5320 return Register();
5321}
5322
5323Register fastEmit_ISD_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5324 if (RetVT.SimpleTy != MVT::f64)
5325 return Register();
5326 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
5327 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMD, RC: &ARM::DPRRegClass, Op0, Op1);
5328 }
5329 return Register();
5330}
5331
5332Register fastEmit_ISD_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5333 if (RetVT.SimpleTy != MVT::v4f16)
5334 return Register();
5335 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5336 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNDh, RC: &ARM::DPRRegClass, Op0, Op1);
5337 }
5338 return Register();
5339}
5340
5341Register fastEmit_ISD_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5342 if (RetVT.SimpleTy != MVT::v8f16)
5343 return Register();
5344 if ((Subtarget->hasMVEFloatOps())) {
5345 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINNMf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5346 }
5347 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5348 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNQh, RC: &ARM::QPRRegClass, Op0, Op1);
5349 }
5350 return Register();
5351}
5352
5353Register fastEmit_ISD_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5354 if (RetVT.SimpleTy != MVT::v2f32)
5355 return Register();
5356 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
5357 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNDf, RC: &ARM::DPRRegClass, Op0, Op1);
5358 }
5359 return Register();
5360}
5361
5362Register fastEmit_ISD_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5363 if (RetVT.SimpleTy != MVT::v4f32)
5364 return Register();
5365 if ((Subtarget->hasMVEFloatOps())) {
5366 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINNMf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5367 }
5368 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
5369 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNQf, RC: &ARM::QPRRegClass, Op0, Op1);
5370 }
5371 return Register();
5372}
5373
5374Register fastEmit_ISD_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5375 switch (VT.SimpleTy) {
5376 case MVT::f16: return fastEmit_ISD_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
5377 case MVT::f32: return fastEmit_ISD_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
5378 case MVT::f64: return fastEmit_ISD_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
5379 case MVT::v4f16: return fastEmit_ISD_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
5380 case MVT::v8f16: return fastEmit_ISD_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
5381 case MVT::v2f32: return fastEmit_ISD_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
5382 case MVT::v4f32: return fastEmit_ISD_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
5383 default: return Register();
5384 }
5385}
5386
5387// FastEmit functions for ISD::FMUL.
5388
5389Register fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5390 if (RetVT.SimpleTy != MVT::f16)
5391 return Register();
5392 if ((Subtarget->hasFullFP16())) {
5393 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULH, RC: &ARM::HPRRegClass, Op0, Op1);
5394 }
5395 return Register();
5396}
5397
5398Register fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5399 if (RetVT.SimpleTy != MVT::f32)
5400 return Register();
5401 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
5402 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULS, RC: &ARM::SPRRegClass, Op0, Op1);
5403 }
5404 return Register();
5405}
5406
5407Register fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5408 if (RetVT.SimpleTy != MVT::f64)
5409 return Register();
5410 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
5411 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULD, RC: &ARM::DPRRegClass, Op0, Op1);
5412 }
5413 return Register();
5414}
5415
5416Register fastEmit_ISD_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5417 if (RetVT.SimpleTy != MVT::v4f16)
5418 return Register();
5419 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5420 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULhd, RC: &ARM::DPRRegClass, Op0, Op1);
5421 }
5422 return Register();
5423}
5424
5425Register fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5426 if (RetVT.SimpleTy != MVT::v8f16)
5427 return Register();
5428 if ((Subtarget->hasMVEFloatOps())) {
5429 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5430 }
5431 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5432 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULhq, RC: &ARM::QPRRegClass, Op0, Op1);
5433 }
5434 return Register();
5435}
5436
5437Register fastEmit_ISD_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5438 if (RetVT.SimpleTy != MVT::v2f32)
5439 return Register();
5440 if ((Subtarget->hasNEON())) {
5441 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULfd, RC: &ARM::DPRRegClass, Op0, Op1);
5442 }
5443 return Register();
5444}
5445
5446Register fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5447 if (RetVT.SimpleTy != MVT::v4f32)
5448 return Register();
5449 if ((Subtarget->hasMVEFloatOps())) {
5450 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5451 }
5452 if ((Subtarget->hasNEON())) {
5453 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULfq, RC: &ARM::QPRRegClass, Op0, Op1);
5454 }
5455 return Register();
5456}
5457
5458Register fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5459 switch (VT.SimpleTy) {
5460 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
5461 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
5462 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
5463 case MVT::v4f16: return fastEmit_ISD_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
5464 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
5465 case MVT::v2f32: return fastEmit_ISD_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
5466 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
5467 default: return Register();
5468 }
5469}
5470
5471// FastEmit functions for ISD::FSUB.
5472
5473Register fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5474 if (RetVT.SimpleTy != MVT::f16)
5475 return Register();
5476 if ((Subtarget->hasFullFP16())) {
5477 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBH, RC: &ARM::HPRRegClass, Op0, Op1);
5478 }
5479 return Register();
5480}
5481
5482Register fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5483 if (RetVT.SimpleTy != MVT::f32)
5484 return Register();
5485 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
5486 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBS, RC: &ARM::SPRRegClass, Op0, Op1);
5487 }
5488 return Register();
5489}
5490
5491Register fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5492 if (RetVT.SimpleTy != MVT::f64)
5493 return Register();
5494 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
5495 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBD, RC: &ARM::DPRRegClass, Op0, Op1);
5496 }
5497 return Register();
5498}
5499
5500Register fastEmit_ISD_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5501 if (RetVT.SimpleTy != MVT::v4f16)
5502 return Register();
5503 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5504 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBhd, RC: &ARM::DPRRegClass, Op0, Op1);
5505 }
5506 return Register();
5507}
5508
5509Register fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5510 if (RetVT.SimpleTy != MVT::v8f16)
5511 return Register();
5512 if ((Subtarget->hasMVEFloatOps())) {
5513 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5514 }
5515 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5516 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBhq, RC: &ARM::QPRRegClass, Op0, Op1);
5517 }
5518 return Register();
5519}
5520
5521Register fastEmit_ISD_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5522 if (RetVT.SimpleTy != MVT::v2f32)
5523 return Register();
5524 if ((Subtarget->hasNEON())) {
5525 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBfd, RC: &ARM::DPRRegClass, Op0, Op1);
5526 }
5527 return Register();
5528}
5529
5530Register fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5531 if (RetVT.SimpleTy != MVT::v4f32)
5532 return Register();
5533 if ((Subtarget->hasMVEFloatOps())) {
5534 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5535 }
5536 if ((Subtarget->hasNEON())) {
5537 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBfq, RC: &ARM::QPRRegClass, Op0, Op1);
5538 }
5539 return Register();
5540}
5541
5542Register fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5543 switch (VT.SimpleTy) {
5544 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
5545 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
5546 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
5547 case MVT::v4f16: return fastEmit_ISD_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
5548 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
5549 case MVT::v2f32: return fastEmit_ISD_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
5550 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
5551 default: return Register();
5552 }
5553}
5554
5555// FastEmit functions for ISD::MUL.
5556
5557Register fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5558 if (RetVT.SimpleTy != MVT::i32)
5559 return Register();
5560 if ((Subtarget->isThumb2())) {
5561 return fastEmitInst_rr(MachineInstOpcode: ARM::t2MUL, RC: &ARM::rGPRRegClass, Op0, Op1);
5562 }
5563 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5564 return fastEmitInst_rr(MachineInstOpcode: ARM::tMUL, RC: &ARM::tGPRRegClass, Op0, Op1);
5565 }
5566 if ((!Subtarget->isThumb()) && (!Subtarget->hasV6Ops()) && (Subtarget->useMulOps())) {
5567 return fastEmitInst_rr(MachineInstOpcode: ARM::MULv5, RC: &ARM::GPRnopcRegClass, Op0, Op1);
5568 }
5569 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
5570 return fastEmitInst_rr(MachineInstOpcode: ARM::MUL, RC: &ARM::GPRnopcRegClass, Op0, Op1);
5571 }
5572 return Register();
5573}
5574
5575Register fastEmit_ISD_MUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
5576 if (RetVT.SimpleTy != MVT::v8i8)
5577 return Register();
5578 if ((Subtarget->hasNEON())) {
5579 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
5580 }
5581 return Register();
5582}
5583
5584Register fastEmit_ISD_MUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5585 if (RetVT.SimpleTy != MVT::v16i8)
5586 return Register();
5587 if ((Subtarget->hasMVEIntegerOps())) {
5588 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULi8, RC: &ARM::MQPRRegClass, Op0, Op1);
5589 }
5590 if ((Subtarget->hasNEON())) {
5591 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
5592 }
5593 return Register();
5594}
5595
5596Register fastEmit_ISD_MUL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
5597 if (RetVT.SimpleTy != MVT::v4i16)
5598 return Register();
5599 if ((Subtarget->hasNEON())) {
5600 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
5601 }
5602 return Register();
5603}
5604
5605Register fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5606 if (RetVT.SimpleTy != MVT::v8i16)
5607 return Register();
5608 if ((Subtarget->hasMVEIntegerOps())) {
5609 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULi16, RC: &ARM::MQPRRegClass, Op0, Op1);
5610 }
5611 if ((Subtarget->hasNEON())) {
5612 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
5613 }
5614 return Register();
5615}
5616
5617Register fastEmit_ISD_MUL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
5618 if (RetVT.SimpleTy != MVT::v2i32)
5619 return Register();
5620 if ((Subtarget->hasNEON())) {
5621 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
5622 }
5623 return Register();
5624}
5625
5626Register fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5627 if (RetVT.SimpleTy != MVT::v4i32)
5628 return Register();
5629 if ((Subtarget->hasMVEIntegerOps())) {
5630 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULi32, RC: &ARM::MQPRRegClass, Op0, Op1);
5631 }
5632 if ((Subtarget->hasNEON())) {
5633 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
5634 }
5635 return Register();
5636}
5637
5638Register fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5639 switch (VT.SimpleTy) {
5640 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op1);
5641 case MVT::v8i8: return fastEmit_ISD_MUL_MVT_v8i8_rr(RetVT, Op0, Op1);
5642 case MVT::v16i8: return fastEmit_ISD_MUL_MVT_v16i8_rr(RetVT, Op0, Op1);
5643 case MVT::v4i16: return fastEmit_ISD_MUL_MVT_v4i16_rr(RetVT, Op0, Op1);
5644 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
5645 case MVT::v2i32: return fastEmit_ISD_MUL_MVT_v2i32_rr(RetVT, Op0, Op1);
5646 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
5647 default: return Register();
5648 }
5649}
5650
5651// FastEmit functions for ISD::MULHS.
5652
5653Register fastEmit_ISD_MULHS_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5654 if (RetVT.SimpleTy != MVT::i32)
5655 return Register();
5656 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
5657 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SMMUL, RC: &ARM::rGPRRegClass, Op0, Op1);
5658 }
5659 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
5660 return fastEmitInst_rr(MachineInstOpcode: ARM::SMMUL, RC: &ARM::GPRRegClass, Op0, Op1);
5661 }
5662 return Register();
5663}
5664
5665Register fastEmit_ISD_MULHS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5666 if (RetVT.SimpleTy != MVT::v16i8)
5667 return Register();
5668 if ((Subtarget->hasMVEIntegerOps())) {
5669 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHs8, RC: &ARM::MQPRRegClass, Op0, Op1);
5670 }
5671 return Register();
5672}
5673
5674Register fastEmit_ISD_MULHS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5675 if (RetVT.SimpleTy != MVT::v8i16)
5676 return Register();
5677 if ((Subtarget->hasMVEIntegerOps())) {
5678 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHs16, RC: &ARM::MQPRRegClass, Op0, Op1);
5679 }
5680 return Register();
5681}
5682
5683Register fastEmit_ISD_MULHS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5684 if (RetVT.SimpleTy != MVT::v4i32)
5685 return Register();
5686 if ((Subtarget->hasMVEIntegerOps())) {
5687 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHs32, RC: &ARM::MQPRRegClass, Op0, Op1);
5688 }
5689 return Register();
5690}
5691
5692Register fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5693 switch (VT.SimpleTy) {
5694 case MVT::i32: return fastEmit_ISD_MULHS_MVT_i32_rr(RetVT, Op0, Op1);
5695 case MVT::v16i8: return fastEmit_ISD_MULHS_MVT_v16i8_rr(RetVT, Op0, Op1);
5696 case MVT::v8i16: return fastEmit_ISD_MULHS_MVT_v8i16_rr(RetVT, Op0, Op1);
5697 case MVT::v4i32: return fastEmit_ISD_MULHS_MVT_v4i32_rr(RetVT, Op0, Op1);
5698 default: return Register();
5699 }
5700}
5701
5702// FastEmit functions for ISD::MULHU.
5703
5704Register fastEmit_ISD_MULHU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5705 if (RetVT.SimpleTy != MVT::v16i8)
5706 return Register();
5707 if ((Subtarget->hasMVEIntegerOps())) {
5708 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHu8, RC: &ARM::MQPRRegClass, Op0, Op1);
5709 }
5710 return Register();
5711}
5712
5713Register fastEmit_ISD_MULHU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5714 if (RetVT.SimpleTy != MVT::v8i16)
5715 return Register();
5716 if ((Subtarget->hasMVEIntegerOps())) {
5717 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHu16, RC: &ARM::MQPRRegClass, Op0, Op1);
5718 }
5719 return Register();
5720}
5721
5722Register fastEmit_ISD_MULHU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5723 if (RetVT.SimpleTy != MVT::v4i32)
5724 return Register();
5725 if ((Subtarget->hasMVEIntegerOps())) {
5726 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHu32, RC: &ARM::MQPRRegClass, Op0, Op1);
5727 }
5728 return Register();
5729}
5730
5731Register fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5732 switch (VT.SimpleTy) {
5733 case MVT::v16i8: return fastEmit_ISD_MULHU_MVT_v16i8_rr(RetVT, Op0, Op1);
5734 case MVT::v8i16: return fastEmit_ISD_MULHU_MVT_v8i16_rr(RetVT, Op0, Op1);
5735 case MVT::v4i32: return fastEmit_ISD_MULHU_MVT_v4i32_rr(RetVT, Op0, Op1);
5736 default: return Register();
5737 }
5738}
5739
5740// FastEmit functions for ISD::OR.
5741
5742Register fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5743 if (RetVT.SimpleTy != MVT::i32)
5744 return Register();
5745 if ((Subtarget->isThumb2())) {
5746 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ORRrr, RC: &ARM::rGPRRegClass, Op0, Op1);
5747 }
5748 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5749 return fastEmitInst_rr(MachineInstOpcode: ARM::tORR, RC: &ARM::tGPRRegClass, Op0, Op1);
5750 }
5751 if ((!Subtarget->isThumb())) {
5752 return fastEmitInst_rr(MachineInstOpcode: ARM::ORRrr, RC: &ARM::GPRRegClass, Op0, Op1);
5753 }
5754 return Register();
5755}
5756
5757Register fastEmit_ISD_OR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
5758 if (RetVT.SimpleTy != MVT::v8i8)
5759 return Register();
5760 if ((Subtarget->hasNEON())) {
5761 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
5762 }
5763 return Register();
5764}
5765
5766Register fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5767 if (RetVT.SimpleTy != MVT::v16i8)
5768 return Register();
5769 if ((Subtarget->hasMVEIntegerOps())) {
5770 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
5771 }
5772 if ((Subtarget->hasNEON())) {
5773 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
5774 }
5775 return Register();
5776}
5777
5778Register fastEmit_ISD_OR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
5779 if (RetVT.SimpleTy != MVT::v4i16)
5780 return Register();
5781 if ((Subtarget->hasNEON())) {
5782 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
5783 }
5784 return Register();
5785}
5786
5787Register fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5788 if (RetVT.SimpleTy != MVT::v8i16)
5789 return Register();
5790 if ((Subtarget->hasMVEIntegerOps())) {
5791 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
5792 }
5793 if ((Subtarget->hasNEON())) {
5794 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
5795 }
5796 return Register();
5797}
5798
5799Register fastEmit_ISD_OR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
5800 if (RetVT.SimpleTy != MVT::v2i32)
5801 return Register();
5802 if ((Subtarget->hasNEON())) {
5803 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
5804 }
5805 return Register();
5806}
5807
5808Register fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5809 if (RetVT.SimpleTy != MVT::v4i32)
5810 return Register();
5811 if ((Subtarget->hasMVEIntegerOps())) {
5812 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
5813 }
5814 if ((Subtarget->hasNEON())) {
5815 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
5816 }
5817 return Register();
5818}
5819
5820Register fastEmit_ISD_OR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
5821 if (RetVT.SimpleTy != MVT::v1i64)
5822 return Register();
5823 if ((Subtarget->hasNEON())) {
5824 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
5825 }
5826 return Register();
5827}
5828
5829Register fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
5830 if (RetVT.SimpleTy != MVT::v2i64)
5831 return Register();
5832 if ((Subtarget->hasMVEIntegerOps())) {
5833 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
5834 }
5835 if ((Subtarget->hasNEON())) {
5836 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
5837 }
5838 return Register();
5839}
5840
5841Register fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5842 switch (VT.SimpleTy) {
5843 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
5844 case MVT::v8i8: return fastEmit_ISD_OR_MVT_v8i8_rr(RetVT, Op0, Op1);
5845 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
5846 case MVT::v4i16: return fastEmit_ISD_OR_MVT_v4i16_rr(RetVT, Op0, Op1);
5847 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
5848 case MVT::v2i32: return fastEmit_ISD_OR_MVT_v2i32_rr(RetVT, Op0, Op1);
5849 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
5850 case MVT::v1i64: return fastEmit_ISD_OR_MVT_v1i64_rr(RetVT, Op0, Op1);
5851 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
5852 default: return Register();
5853 }
5854}
5855
5856// FastEmit functions for ISD::ROTR.
5857
5858Register fastEmit_ISD_ROTR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5859 if (RetVT.SimpleTy != MVT::i32)
5860 return Register();
5861 if ((Subtarget->isThumb2())) {
5862 return fastEmitInst_rr(MachineInstOpcode: ARM::t2RORrr, RC: &ARM::rGPRRegClass, Op0, Op1);
5863 }
5864 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5865 return fastEmitInst_rr(MachineInstOpcode: ARM::tROR, RC: &ARM::tGPRRegClass, Op0, Op1);
5866 }
5867 return Register();
5868}
5869
5870Register fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5871 switch (VT.SimpleTy) {
5872 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_rr(RetVT, Op0, Op1);
5873 default: return Register();
5874 }
5875}
5876
5877// FastEmit functions for ISD::SADDSAT.
5878
5879Register fastEmit_ISD_SADDSAT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5880 if (RetVT.SimpleTy != MVT::i32)
5881 return Register();
5882 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
5883 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QADD, RC: &ARM::rGPRRegClass, Op0, Op1);
5884 }
5885 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
5886 return fastEmitInst_rr(MachineInstOpcode: ARM::QADD, RC: &ARM::GPRnopcRegClass, Op0, Op1);
5887 }
5888 return Register();
5889}
5890
5891Register fastEmit_ISD_SADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
5892 if (RetVT.SimpleTy != MVT::v8i8)
5893 return Register();
5894 if ((Subtarget->hasNEON())) {
5895 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
5896 }
5897 return Register();
5898}
5899
5900Register fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5901 if (RetVT.SimpleTy != MVT::v16i8)
5902 return Register();
5903 if ((Subtarget->hasMVEIntegerOps())) {
5904 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
5905 }
5906 if ((Subtarget->hasNEON())) {
5907 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
5908 }
5909 return Register();
5910}
5911
5912Register fastEmit_ISD_SADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
5913 if (RetVT.SimpleTy != MVT::v4i16)
5914 return Register();
5915 if ((Subtarget->hasNEON())) {
5916 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
5917 }
5918 return Register();
5919}
5920
5921Register fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5922 if (RetVT.SimpleTy != MVT::v8i16)
5923 return Register();
5924 if ((Subtarget->hasMVEIntegerOps())) {
5925 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
5926 }
5927 if ((Subtarget->hasNEON())) {
5928 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
5929 }
5930 return Register();
5931}
5932
5933Register fastEmit_ISD_SADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
5934 if (RetVT.SimpleTy != MVT::v2i32)
5935 return Register();
5936 if ((Subtarget->hasNEON())) {
5937 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
5938 }
5939 return Register();
5940}
5941
5942Register fastEmit_ISD_SADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5943 if (RetVT.SimpleTy != MVT::v4i32)
5944 return Register();
5945 if ((Subtarget->hasMVEIntegerOps())) {
5946 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
5947 }
5948 if ((Subtarget->hasNEON())) {
5949 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
5950 }
5951 return Register();
5952}
5953
5954Register fastEmit_ISD_SADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
5955 if (RetVT.SimpleTy != MVT::v1i64)
5956 return Register();
5957 if ((Subtarget->hasNEON())) {
5958 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
5959 }
5960 return Register();
5961}
5962
5963Register fastEmit_ISD_SADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
5964 if (RetVT.SimpleTy != MVT::v2i64)
5965 return Register();
5966 if ((Subtarget->hasNEON())) {
5967 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
5968 }
5969 return Register();
5970}
5971
5972Register fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5973 switch (VT.SimpleTy) {
5974 case MVT::i32: return fastEmit_ISD_SADDSAT_MVT_i32_rr(RetVT, Op0, Op1);
5975 case MVT::v8i8: return fastEmit_ISD_SADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
5976 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
5977 case MVT::v4i16: return fastEmit_ISD_SADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
5978 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
5979 case MVT::v2i32: return fastEmit_ISD_SADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
5980 case MVT::v4i32: return fastEmit_ISD_SADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
5981 case MVT::v1i64: return fastEmit_ISD_SADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
5982 case MVT::v2i64: return fastEmit_ISD_SADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
5983 default: return Register();
5984 }
5985}
5986
5987// FastEmit functions for ISD::SDIV.
5988
5989Register fastEmit_ISD_SDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5990 if (RetVT.SimpleTy != MVT::i32)
5991 return Register();
5992 if ((Subtarget->hasDivideInThumbMode()) && (Subtarget->hasV8MBaselineOps()) && (Subtarget->isThumb())) {
5993 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SDIV, RC: &ARM::rGPRRegClass, Op0, Op1);
5994 }
5995 if ((Subtarget->hasDivideInARMMode()) && (!Subtarget->isThumb())) {
5996 return fastEmitInst_rr(MachineInstOpcode: ARM::SDIV, RC: &ARM::GPRRegClass, Op0, Op1);
5997 }
5998 return Register();
5999}
6000
6001Register fastEmit_ISD_SDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6002 switch (VT.SimpleTy) {
6003 case MVT::i32: return fastEmit_ISD_SDIV_MVT_i32_rr(RetVT, Op0, Op1);
6004 default: return Register();
6005 }
6006}
6007
6008// FastEmit functions for ISD::SHL.
6009
6010Register fastEmit_ISD_SHL_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6011 if (RetVT.SimpleTy != MVT::i32)
6012 return Register();
6013 if ((Subtarget->isThumb2())) {
6014 return fastEmitInst_rr(MachineInstOpcode: ARM::t2LSLrr, RC: &ARM::rGPRRegClass, Op0, Op1);
6015 }
6016 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6017 return fastEmitInst_rr(MachineInstOpcode: ARM::tLSLrr, RC: &ARM::tGPRRegClass, Op0, Op1);
6018 }
6019 return Register();
6020}
6021
6022Register fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6023 switch (VT.SimpleTy) {
6024 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_rr(RetVT, Op0, Op1);
6025 default: return Register();
6026 }
6027}
6028
6029// FastEmit functions for ISD::SMAX.
6030
6031Register fastEmit_ISD_SMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6032 if (RetVT.SimpleTy != MVT::v8i8)
6033 return Register();
6034 if ((Subtarget->hasNEON())) {
6035 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6036 }
6037 return Register();
6038}
6039
6040Register fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6041 if (RetVT.SimpleTy != MVT::v16i8)
6042 return Register();
6043 if ((Subtarget->hasMVEIntegerOps())) {
6044 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXs8, RC: &ARM::MQPRRegClass, Op0, Op1);
6045 }
6046 if ((Subtarget->hasNEON())) {
6047 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6048 }
6049 return Register();
6050}
6051
6052Register fastEmit_ISD_SMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6053 if (RetVT.SimpleTy != MVT::v4i16)
6054 return Register();
6055 if ((Subtarget->hasNEON())) {
6056 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6057 }
6058 return Register();
6059}
6060
6061Register fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6062 if (RetVT.SimpleTy != MVT::v8i16)
6063 return Register();
6064 if ((Subtarget->hasMVEIntegerOps())) {
6065 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXs16, RC: &ARM::MQPRRegClass, Op0, Op1);
6066 }
6067 if ((Subtarget->hasNEON())) {
6068 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6069 }
6070 return Register();
6071}
6072
6073Register fastEmit_ISD_SMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6074 if (RetVT.SimpleTy != MVT::v2i32)
6075 return Register();
6076 if ((Subtarget->hasNEON())) {
6077 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6078 }
6079 return Register();
6080}
6081
6082Register fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6083 if (RetVT.SimpleTy != MVT::v4i32)
6084 return Register();
6085 if ((Subtarget->hasMVEIntegerOps())) {
6086 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXs32, RC: &ARM::MQPRRegClass, Op0, Op1);
6087 }
6088 if ((Subtarget->hasNEON())) {
6089 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6090 }
6091 return Register();
6092}
6093
6094Register fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6095 switch (VT.SimpleTy) {
6096 case MVT::v8i8: return fastEmit_ISD_SMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
6097 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
6098 case MVT::v4i16: return fastEmit_ISD_SMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
6099 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
6100 case MVT::v2i32: return fastEmit_ISD_SMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
6101 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
6102 default: return Register();
6103 }
6104}
6105
6106// FastEmit functions for ISD::SMIN.
6107
6108Register fastEmit_ISD_SMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6109 if (RetVT.SimpleTy != MVT::v8i8)
6110 return Register();
6111 if ((Subtarget->hasNEON())) {
6112 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6113 }
6114 return Register();
6115}
6116
6117Register fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6118 if (RetVT.SimpleTy != MVT::v16i8)
6119 return Register();
6120 if ((Subtarget->hasMVEIntegerOps())) {
6121 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINs8, RC: &ARM::MQPRRegClass, Op0, Op1);
6122 }
6123 if ((Subtarget->hasNEON())) {
6124 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6125 }
6126 return Register();
6127}
6128
6129Register fastEmit_ISD_SMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6130 if (RetVT.SimpleTy != MVT::v4i16)
6131 return Register();
6132 if ((Subtarget->hasNEON())) {
6133 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6134 }
6135 return Register();
6136}
6137
6138Register fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6139 if (RetVT.SimpleTy != MVT::v8i16)
6140 return Register();
6141 if ((Subtarget->hasMVEIntegerOps())) {
6142 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINs16, RC: &ARM::MQPRRegClass, Op0, Op1);
6143 }
6144 if ((Subtarget->hasNEON())) {
6145 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6146 }
6147 return Register();
6148}
6149
6150Register fastEmit_ISD_SMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6151 if (RetVT.SimpleTy != MVT::v2i32)
6152 return Register();
6153 if ((Subtarget->hasNEON())) {
6154 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6155 }
6156 return Register();
6157}
6158
6159Register fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6160 if (RetVT.SimpleTy != MVT::v4i32)
6161 return Register();
6162 if ((Subtarget->hasMVEIntegerOps())) {
6163 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINs32, RC: &ARM::MQPRRegClass, Op0, Op1);
6164 }
6165 if ((Subtarget->hasNEON())) {
6166 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6167 }
6168 return Register();
6169}
6170
6171Register fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6172 switch (VT.SimpleTy) {
6173 case MVT::v8i8: return fastEmit_ISD_SMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
6174 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
6175 case MVT::v4i16: return fastEmit_ISD_SMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
6176 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
6177 case MVT::v2i32: return fastEmit_ISD_SMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
6178 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
6179 default: return Register();
6180 }
6181}
6182
6183// FastEmit functions for ISD::SRA.
6184
6185Register fastEmit_ISD_SRA_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6186 if (RetVT.SimpleTy != MVT::i32)
6187 return Register();
6188 if ((Subtarget->isThumb2())) {
6189 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ASRrr, RC: &ARM::rGPRRegClass, Op0, Op1);
6190 }
6191 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6192 return fastEmitInst_rr(MachineInstOpcode: ARM::tASRrr, RC: &ARM::tGPRRegClass, Op0, Op1);
6193 }
6194 return Register();
6195}
6196
6197Register fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6198 switch (VT.SimpleTy) {
6199 case MVT::i32: return fastEmit_ISD_SRA_MVT_i32_rr(RetVT, Op0, Op1);
6200 default: return Register();
6201 }
6202}
6203
6204// FastEmit functions for ISD::SRL.
6205
6206Register fastEmit_ISD_SRL_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6207 if (RetVT.SimpleTy != MVT::i32)
6208 return Register();
6209 if ((Subtarget->isThumb2())) {
6210 return fastEmitInst_rr(MachineInstOpcode: ARM::t2LSRrr, RC: &ARM::rGPRRegClass, Op0, Op1);
6211 }
6212 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6213 return fastEmitInst_rr(MachineInstOpcode: ARM::tLSRrr, RC: &ARM::tGPRRegClass, Op0, Op1);
6214 }
6215 return Register();
6216}
6217
6218Register fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6219 switch (VT.SimpleTy) {
6220 case MVT::i32: return fastEmit_ISD_SRL_MVT_i32_rr(RetVT, Op0, Op1);
6221 default: return Register();
6222 }
6223}
6224
6225// FastEmit functions for ISD::SSUBSAT.
6226
6227Register fastEmit_ISD_SSUBSAT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6228 if (RetVT.SimpleTy != MVT::i32)
6229 return Register();
6230 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
6231 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QSUB, RC: &ARM::rGPRRegClass, Op0, Op1);
6232 }
6233 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
6234 return fastEmitInst_rr(MachineInstOpcode: ARM::QSUB, RC: &ARM::GPRnopcRegClass, Op0, Op1);
6235 }
6236 return Register();
6237}
6238
6239Register fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6240 if (RetVT.SimpleTy != MVT::v8i8)
6241 return Register();
6242 if ((Subtarget->hasNEON())) {
6243 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6244 }
6245 return Register();
6246}
6247
6248Register fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6249 if (RetVT.SimpleTy != MVT::v16i8)
6250 return Register();
6251 if ((Subtarget->hasMVEIntegerOps())) {
6252 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBs8, RC: &ARM::MQPRRegClass, Op0, Op1);
6253 }
6254 if ((Subtarget->hasNEON())) {
6255 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6256 }
6257 return Register();
6258}
6259
6260Register fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6261 if (RetVT.SimpleTy != MVT::v4i16)
6262 return Register();
6263 if ((Subtarget->hasNEON())) {
6264 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6265 }
6266 return Register();
6267}
6268
6269Register fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6270 if (RetVT.SimpleTy != MVT::v8i16)
6271 return Register();
6272 if ((Subtarget->hasMVEIntegerOps())) {
6273 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBs16, RC: &ARM::MQPRRegClass, Op0, Op1);
6274 }
6275 if ((Subtarget->hasNEON())) {
6276 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6277 }
6278 return Register();
6279}
6280
6281Register fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6282 if (RetVT.SimpleTy != MVT::v2i32)
6283 return Register();
6284 if ((Subtarget->hasNEON())) {
6285 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6286 }
6287 return Register();
6288}
6289
6290Register fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6291 if (RetVT.SimpleTy != MVT::v4i32)
6292 return Register();
6293 if ((Subtarget->hasMVEIntegerOps())) {
6294 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBs32, RC: &ARM::MQPRRegClass, Op0, Op1);
6295 }
6296 if ((Subtarget->hasNEON())) {
6297 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6298 }
6299 return Register();
6300}
6301
6302Register fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
6303 if (RetVT.SimpleTy != MVT::v1i64)
6304 return Register();
6305 if ((Subtarget->hasNEON())) {
6306 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
6307 }
6308 return Register();
6309}
6310
6311Register fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6312 if (RetVT.SimpleTy != MVT::v2i64)
6313 return Register();
6314 if ((Subtarget->hasNEON())) {
6315 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
6316 }
6317 return Register();
6318}
6319
6320Register fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6321 switch (VT.SimpleTy) {
6322 case MVT::i32: return fastEmit_ISD_SSUBSAT_MVT_i32_rr(RetVT, Op0, Op1);
6323 case MVT::v8i8: return fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
6324 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
6325 case MVT::v4i16: return fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
6326 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
6327 case MVT::v2i32: return fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
6328 case MVT::v4i32: return fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
6329 case MVT::v1i64: return fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
6330 case MVT::v2i64: return fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
6331 default: return Register();
6332 }
6333}
6334
6335// FastEmit functions for ISD::STRICT_FADD.
6336
6337Register fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6338 if (RetVT.SimpleTy != MVT::f16)
6339 return Register();
6340 if ((Subtarget->hasFullFP16())) {
6341 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDH, RC: &ARM::HPRRegClass, Op0, Op1);
6342 }
6343 return Register();
6344}
6345
6346Register fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6347 if (RetVT.SimpleTy != MVT::f32)
6348 return Register();
6349 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
6350 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDS, RC: &ARM::SPRRegClass, Op0, Op1);
6351 }
6352 return Register();
6353}
6354
6355Register fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6356 if (RetVT.SimpleTy != MVT::f64)
6357 return Register();
6358 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
6359 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDD, RC: &ARM::DPRRegClass, Op0, Op1);
6360 }
6361 return Register();
6362}
6363
6364Register fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6365 switch (VT.SimpleTy) {
6366 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1);
6367 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1);
6368 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1);
6369 default: return Register();
6370 }
6371}
6372
6373// FastEmit functions for ISD::STRICT_FDIV.
6374
6375Register fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6376 if (RetVT.SimpleTy != MVT::f16)
6377 return Register();
6378 if ((Subtarget->hasFullFP16())) {
6379 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVH, RC: &ARM::HPRRegClass, Op0, Op1);
6380 }
6381 return Register();
6382}
6383
6384Register fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6385 if (RetVT.SimpleTy != MVT::f32)
6386 return Register();
6387 if ((Subtarget->hasVFP2Base())) {
6388 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVS, RC: &ARM::SPRRegClass, Op0, Op1);
6389 }
6390 return Register();
6391}
6392
6393Register fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6394 if (RetVT.SimpleTy != MVT::f64)
6395 return Register();
6396 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
6397 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVD, RC: &ARM::DPRRegClass, Op0, Op1);
6398 }
6399 return Register();
6400}
6401
6402Register fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6403 switch (VT.SimpleTy) {
6404 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
6405 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
6406 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
6407 default: return Register();
6408 }
6409}
6410
6411// FastEmit functions for ISD::STRICT_FMAXNUM.
6412
6413Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6414 if (RetVT.SimpleTy != MVT::f16)
6415 return Register();
6416 if ((Subtarget->hasFullFP16())) {
6417 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMH, RC: &ARM::HPRRegClass, Op0, Op1);
6418 }
6419 return Register();
6420}
6421
6422Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6423 if (RetVT.SimpleTy != MVT::f32)
6424 return Register();
6425 if ((Subtarget->hasFPARMv8Base())) {
6426 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMS, RC: &ARM::SPRRegClass, Op0, Op1);
6427 }
6428 return Register();
6429}
6430
6431Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6432 if (RetVT.SimpleTy != MVT::f64)
6433 return Register();
6434 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
6435 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMD, RC: &ARM::DPRRegClass, Op0, Op1);
6436 }
6437 return Register();
6438}
6439
6440Register fastEmit_ISD_STRICT_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6441 switch (VT.SimpleTy) {
6442 case MVT::f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
6443 case MVT::f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
6444 case MVT::f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
6445 default: return Register();
6446 }
6447}
6448
6449// FastEmit functions for ISD::STRICT_FMINNUM.
6450
6451Register fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6452 if (RetVT.SimpleTy != MVT::f16)
6453 return Register();
6454 if ((Subtarget->hasFullFP16())) {
6455 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMH, RC: &ARM::HPRRegClass, Op0, Op1);
6456 }
6457 return Register();
6458}
6459
6460Register fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6461 if (RetVT.SimpleTy != MVT::f32)
6462 return Register();
6463 if ((Subtarget->hasFPARMv8Base())) {
6464 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMS, RC: &ARM::SPRRegClass, Op0, Op1);
6465 }
6466 return Register();
6467}
6468
6469Register fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6470 if (RetVT.SimpleTy != MVT::f64)
6471 return Register();
6472 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
6473 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMD, RC: &ARM::DPRRegClass, Op0, Op1);
6474 }
6475 return Register();
6476}
6477
6478Register fastEmit_ISD_STRICT_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6479 switch (VT.SimpleTy) {
6480 case MVT::f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
6481 case MVT::f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
6482 case MVT::f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
6483 default: return Register();
6484 }
6485}
6486
6487// FastEmit functions for ISD::STRICT_FMUL.
6488
6489Register fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6490 if (RetVT.SimpleTy != MVT::f16)
6491 return Register();
6492 if ((Subtarget->hasFullFP16())) {
6493 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULH, RC: &ARM::HPRRegClass, Op0, Op1);
6494 }
6495 return Register();
6496}
6497
6498Register fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6499 if (RetVT.SimpleTy != MVT::f32)
6500 return Register();
6501 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
6502 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULS, RC: &ARM::SPRRegClass, Op0, Op1);
6503 }
6504 return Register();
6505}
6506
6507Register fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6508 if (RetVT.SimpleTy != MVT::f64)
6509 return Register();
6510 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
6511 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULD, RC: &ARM::DPRRegClass, Op0, Op1);
6512 }
6513 return Register();
6514}
6515
6516Register fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6517 switch (VT.SimpleTy) {
6518 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
6519 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
6520 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
6521 default: return Register();
6522 }
6523}
6524
6525// FastEmit functions for ISD::STRICT_FSUB.
6526
6527Register fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6528 if (RetVT.SimpleTy != MVT::f16)
6529 return Register();
6530 if ((Subtarget->hasFullFP16())) {
6531 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBH, RC: &ARM::HPRRegClass, Op0, Op1);
6532 }
6533 return Register();
6534}
6535
6536Register fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6537 if (RetVT.SimpleTy != MVT::f32)
6538 return Register();
6539 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
6540 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBS, RC: &ARM::SPRRegClass, Op0, Op1);
6541 }
6542 return Register();
6543}
6544
6545Register fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6546 if (RetVT.SimpleTy != MVT::f64)
6547 return Register();
6548 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
6549 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBD, RC: &ARM::DPRRegClass, Op0, Op1);
6550 }
6551 return Register();
6552}
6553
6554Register fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6555 switch (VT.SimpleTy) {
6556 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
6557 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
6558 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
6559 default: return Register();
6560 }
6561}
6562
6563// FastEmit functions for ISD::SUB.
6564
6565Register fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6566 if (RetVT.SimpleTy != MVT::i32)
6567 return Register();
6568 if ((Subtarget->isThumb2())) {
6569 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SUBrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
6570 }
6571 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6572 return fastEmitInst_rr(MachineInstOpcode: ARM::tSUBrr, RC: &ARM::tGPRRegClass, Op0, Op1);
6573 }
6574 if ((!Subtarget->isThumb())) {
6575 return fastEmitInst_rr(MachineInstOpcode: ARM::SUBrr, RC: &ARM::GPRRegClass, Op0, Op1);
6576 }
6577 return Register();
6578}
6579
6580Register fastEmit_ISD_SUB_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6581 if (RetVT.SimpleTy != MVT::v8i8)
6582 return Register();
6583 if ((Subtarget->hasNEON())) {
6584 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6585 }
6586 return Register();
6587}
6588
6589Register fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6590 if (RetVT.SimpleTy != MVT::v16i8)
6591 return Register();
6592 if ((Subtarget->hasMVEIntegerOps())) {
6593 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBi8, RC: &ARM::MQPRRegClass, Op0, Op1);
6594 }
6595 if ((Subtarget->hasNEON())) {
6596 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6597 }
6598 return Register();
6599}
6600
6601Register fastEmit_ISD_SUB_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6602 if (RetVT.SimpleTy != MVT::v4i16)
6603 return Register();
6604 if ((Subtarget->hasNEON())) {
6605 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6606 }
6607 return Register();
6608}
6609
6610Register fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6611 if (RetVT.SimpleTy != MVT::v8i16)
6612 return Register();
6613 if ((Subtarget->hasMVEIntegerOps())) {
6614 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBi16, RC: &ARM::MQPRRegClass, Op0, Op1);
6615 }
6616 if ((Subtarget->hasNEON())) {
6617 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6618 }
6619 return Register();
6620}
6621
6622Register fastEmit_ISD_SUB_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6623 if (RetVT.SimpleTy != MVT::v2i32)
6624 return Register();
6625 if ((Subtarget->hasNEON())) {
6626 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6627 }
6628 return Register();
6629}
6630
6631Register fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6632 if (RetVT.SimpleTy != MVT::v4i32)
6633 return Register();
6634 if ((Subtarget->hasMVEIntegerOps())) {
6635 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBi32, RC: &ARM::MQPRRegClass, Op0, Op1);
6636 }
6637 if ((Subtarget->hasNEON())) {
6638 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6639 }
6640 return Register();
6641}
6642
6643Register fastEmit_ISD_SUB_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
6644 if (RetVT.SimpleTy != MVT::v1i64)
6645 return Register();
6646 if ((Subtarget->hasNEON())) {
6647 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
6648 }
6649 return Register();
6650}
6651
6652Register fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6653 if (RetVT.SimpleTy != MVT::v2i64)
6654 return Register();
6655 if ((Subtarget->hasNEON())) {
6656 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
6657 }
6658 return Register();
6659}
6660
6661Register fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6662 switch (VT.SimpleTy) {
6663 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
6664 case MVT::v8i8: return fastEmit_ISD_SUB_MVT_v8i8_rr(RetVT, Op0, Op1);
6665 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
6666 case MVT::v4i16: return fastEmit_ISD_SUB_MVT_v4i16_rr(RetVT, Op0, Op1);
6667 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
6668 case MVT::v2i32: return fastEmit_ISD_SUB_MVT_v2i32_rr(RetVT, Op0, Op1);
6669 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
6670 case MVT::v1i64: return fastEmit_ISD_SUB_MVT_v1i64_rr(RetVT, Op0, Op1);
6671 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
6672 default: return Register();
6673 }
6674}
6675
6676// FastEmit functions for ISD::UADDSAT.
6677
6678Register fastEmit_ISD_UADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6679 if (RetVT.SimpleTy != MVT::v8i8)
6680 return Register();
6681 if ((Subtarget->hasNEON())) {
6682 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6683 }
6684 return Register();
6685}
6686
6687Register fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6688 if (RetVT.SimpleTy != MVT::v16i8)
6689 return Register();
6690 if ((Subtarget->hasMVEIntegerOps())) {
6691 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
6692 }
6693 if ((Subtarget->hasNEON())) {
6694 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6695 }
6696 return Register();
6697}
6698
6699Register fastEmit_ISD_UADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6700 if (RetVT.SimpleTy != MVT::v4i16)
6701 return Register();
6702 if ((Subtarget->hasNEON())) {
6703 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6704 }
6705 return Register();
6706}
6707
6708Register fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6709 if (RetVT.SimpleTy != MVT::v8i16)
6710 return Register();
6711 if ((Subtarget->hasMVEIntegerOps())) {
6712 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
6713 }
6714 if ((Subtarget->hasNEON())) {
6715 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6716 }
6717 return Register();
6718}
6719
6720Register fastEmit_ISD_UADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6721 if (RetVT.SimpleTy != MVT::v2i32)
6722 return Register();
6723 if ((Subtarget->hasNEON())) {
6724 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6725 }
6726 return Register();
6727}
6728
6729Register fastEmit_ISD_UADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6730 if (RetVT.SimpleTy != MVT::v4i32)
6731 return Register();
6732 if ((Subtarget->hasMVEIntegerOps())) {
6733 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
6734 }
6735 if ((Subtarget->hasNEON())) {
6736 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6737 }
6738 return Register();
6739}
6740
6741Register fastEmit_ISD_UADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
6742 if (RetVT.SimpleTy != MVT::v1i64)
6743 return Register();
6744 if ((Subtarget->hasNEON())) {
6745 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
6746 }
6747 return Register();
6748}
6749
6750Register fastEmit_ISD_UADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6751 if (RetVT.SimpleTy != MVT::v2i64)
6752 return Register();
6753 if ((Subtarget->hasNEON())) {
6754 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
6755 }
6756 return Register();
6757}
6758
6759Register fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6760 switch (VT.SimpleTy) {
6761 case MVT::v8i8: return fastEmit_ISD_UADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
6762 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
6763 case MVT::v4i16: return fastEmit_ISD_UADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
6764 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
6765 case MVT::v2i32: return fastEmit_ISD_UADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
6766 case MVT::v4i32: return fastEmit_ISD_UADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
6767 case MVT::v1i64: return fastEmit_ISD_UADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
6768 case MVT::v2i64: return fastEmit_ISD_UADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
6769 default: return Register();
6770 }
6771}
6772
6773// FastEmit functions for ISD::UDIV.
6774
6775Register fastEmit_ISD_UDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6776 if (RetVT.SimpleTy != MVT::i32)
6777 return Register();
6778 if ((Subtarget->hasDivideInThumbMode()) && (Subtarget->hasV8MBaselineOps()) && (Subtarget->isThumb())) {
6779 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UDIV, RC: &ARM::rGPRRegClass, Op0, Op1);
6780 }
6781 if ((Subtarget->hasDivideInARMMode()) && (!Subtarget->isThumb())) {
6782 return fastEmitInst_rr(MachineInstOpcode: ARM::UDIV, RC: &ARM::GPRRegClass, Op0, Op1);
6783 }
6784 return Register();
6785}
6786
6787Register fastEmit_ISD_UDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6788 switch (VT.SimpleTy) {
6789 case MVT::i32: return fastEmit_ISD_UDIV_MVT_i32_rr(RetVT, Op0, Op1);
6790 default: return Register();
6791 }
6792}
6793
6794// FastEmit functions for ISD::UMAX.
6795
6796Register fastEmit_ISD_UMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6797 if (RetVT.SimpleTy != MVT::v8i8)
6798 return Register();
6799 if ((Subtarget->hasNEON())) {
6800 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6801 }
6802 return Register();
6803}
6804
6805Register fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6806 if (RetVT.SimpleTy != MVT::v16i8)
6807 return Register();
6808 if ((Subtarget->hasMVEIntegerOps())) {
6809 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXu8, RC: &ARM::MQPRRegClass, Op0, Op1);
6810 }
6811 if ((Subtarget->hasNEON())) {
6812 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6813 }
6814 return Register();
6815}
6816
6817Register fastEmit_ISD_UMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6818 if (RetVT.SimpleTy != MVT::v4i16)
6819 return Register();
6820 if ((Subtarget->hasNEON())) {
6821 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6822 }
6823 return Register();
6824}
6825
6826Register fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6827 if (RetVT.SimpleTy != MVT::v8i16)
6828 return Register();
6829 if ((Subtarget->hasMVEIntegerOps())) {
6830 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXu16, RC: &ARM::MQPRRegClass, Op0, Op1);
6831 }
6832 if ((Subtarget->hasNEON())) {
6833 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6834 }
6835 return Register();
6836}
6837
6838Register fastEmit_ISD_UMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6839 if (RetVT.SimpleTy != MVT::v2i32)
6840 return Register();
6841 if ((Subtarget->hasNEON())) {
6842 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6843 }
6844 return Register();
6845}
6846
6847Register fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6848 if (RetVT.SimpleTy != MVT::v4i32)
6849 return Register();
6850 if ((Subtarget->hasMVEIntegerOps())) {
6851 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXu32, RC: &ARM::MQPRRegClass, Op0, Op1);
6852 }
6853 if ((Subtarget->hasNEON())) {
6854 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6855 }
6856 return Register();
6857}
6858
6859Register fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6860 switch (VT.SimpleTy) {
6861 case MVT::v8i8: return fastEmit_ISD_UMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
6862 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
6863 case MVT::v4i16: return fastEmit_ISD_UMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
6864 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
6865 case MVT::v2i32: return fastEmit_ISD_UMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
6866 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
6867 default: return Register();
6868 }
6869}
6870
6871// FastEmit functions for ISD::UMIN.
6872
6873Register fastEmit_ISD_UMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6874 if (RetVT.SimpleTy != MVT::v8i8)
6875 return Register();
6876 if ((Subtarget->hasNEON())) {
6877 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6878 }
6879 return Register();
6880}
6881
6882Register fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6883 if (RetVT.SimpleTy != MVT::v16i8)
6884 return Register();
6885 if ((Subtarget->hasMVEIntegerOps())) {
6886 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINu8, RC: &ARM::MQPRRegClass, Op0, Op1);
6887 }
6888 if ((Subtarget->hasNEON())) {
6889 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6890 }
6891 return Register();
6892}
6893
6894Register fastEmit_ISD_UMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6895 if (RetVT.SimpleTy != MVT::v4i16)
6896 return Register();
6897 if ((Subtarget->hasNEON())) {
6898 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6899 }
6900 return Register();
6901}
6902
6903Register fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6904 if (RetVT.SimpleTy != MVT::v8i16)
6905 return Register();
6906 if ((Subtarget->hasMVEIntegerOps())) {
6907 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINu16, RC: &ARM::MQPRRegClass, Op0, Op1);
6908 }
6909 if ((Subtarget->hasNEON())) {
6910 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6911 }
6912 return Register();
6913}
6914
6915Register fastEmit_ISD_UMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6916 if (RetVT.SimpleTy != MVT::v2i32)
6917 return Register();
6918 if ((Subtarget->hasNEON())) {
6919 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6920 }
6921 return Register();
6922}
6923
6924Register fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6925 if (RetVT.SimpleTy != MVT::v4i32)
6926 return Register();
6927 if ((Subtarget->hasMVEIntegerOps())) {
6928 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINu32, RC: &ARM::MQPRRegClass, Op0, Op1);
6929 }
6930 if ((Subtarget->hasNEON())) {
6931 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6932 }
6933 return Register();
6934}
6935
6936Register fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6937 switch (VT.SimpleTy) {
6938 case MVT::v8i8: return fastEmit_ISD_UMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
6939 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
6940 case MVT::v4i16: return fastEmit_ISD_UMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
6941 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
6942 case MVT::v2i32: return fastEmit_ISD_UMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
6943 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
6944 default: return Register();
6945 }
6946}
6947
6948// FastEmit functions for ISD::USUBSAT.
6949
6950Register fastEmit_ISD_USUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6951 if (RetVT.SimpleTy != MVT::v8i8)
6952 return Register();
6953 if ((Subtarget->hasNEON())) {
6954 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6955 }
6956 return Register();
6957}
6958
6959Register fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6960 if (RetVT.SimpleTy != MVT::v16i8)
6961 return Register();
6962 if ((Subtarget->hasMVEIntegerOps())) {
6963 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBu8, RC: &ARM::MQPRRegClass, Op0, Op1);
6964 }
6965 if ((Subtarget->hasNEON())) {
6966 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6967 }
6968 return Register();
6969}
6970
6971Register fastEmit_ISD_USUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6972 if (RetVT.SimpleTy != MVT::v4i16)
6973 return Register();
6974 if ((Subtarget->hasNEON())) {
6975 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6976 }
6977 return Register();
6978}
6979
6980Register fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6981 if (RetVT.SimpleTy != MVT::v8i16)
6982 return Register();
6983 if ((Subtarget->hasMVEIntegerOps())) {
6984 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBu16, RC: &ARM::MQPRRegClass, Op0, Op1);
6985 }
6986 if ((Subtarget->hasNEON())) {
6987 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6988 }
6989 return Register();
6990}
6991
6992Register fastEmit_ISD_USUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6993 if (RetVT.SimpleTy != MVT::v2i32)
6994 return Register();
6995 if ((Subtarget->hasNEON())) {
6996 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6997 }
6998 return Register();
6999}
7000
7001Register fastEmit_ISD_USUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7002 if (RetVT.SimpleTy != MVT::v4i32)
7003 return Register();
7004 if ((Subtarget->hasMVEIntegerOps())) {
7005 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBu32, RC: &ARM::MQPRRegClass, Op0, Op1);
7006 }
7007 if ((Subtarget->hasNEON())) {
7008 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
7009 }
7010 return Register();
7011}
7012
7013Register fastEmit_ISD_USUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7014 if (RetVT.SimpleTy != MVT::v1i64)
7015 return Register();
7016 if ((Subtarget->hasNEON())) {
7017 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
7018 }
7019 return Register();
7020}
7021
7022Register fastEmit_ISD_USUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7023 if (RetVT.SimpleTy != MVT::v2i64)
7024 return Register();
7025 if ((Subtarget->hasNEON())) {
7026 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
7027 }
7028 return Register();
7029}
7030
7031Register fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7032 switch (VT.SimpleTy) {
7033 case MVT::v8i8: return fastEmit_ISD_USUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
7034 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
7035 case MVT::v4i16: return fastEmit_ISD_USUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
7036 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
7037 case MVT::v2i32: return fastEmit_ISD_USUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
7038 case MVT::v4i32: return fastEmit_ISD_USUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
7039 case MVT::v1i64: return fastEmit_ISD_USUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
7040 case MVT::v2i64: return fastEmit_ISD_USUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
7041 default: return Register();
7042 }
7043}
7044
7045// FastEmit functions for ISD::XOR.
7046
7047Register fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
7048 if (RetVT.SimpleTy != MVT::i32)
7049 return Register();
7050 if ((Subtarget->isThumb2())) {
7051 return fastEmitInst_rr(MachineInstOpcode: ARM::t2EORrr, RC: &ARM::rGPRRegClass, Op0, Op1);
7052 }
7053 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
7054 return fastEmitInst_rr(MachineInstOpcode: ARM::tEOR, RC: &ARM::tGPRRegClass, Op0, Op1);
7055 }
7056 if ((!Subtarget->isThumb())) {
7057 return fastEmitInst_rr(MachineInstOpcode: ARM::EORrr, RC: &ARM::GPRRegClass, Op0, Op1);
7058 }
7059 return Register();
7060}
7061
7062Register fastEmit_ISD_XOR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7063 if (RetVT.SimpleTy != MVT::v8i8)
7064 return Register();
7065 if ((Subtarget->hasNEON())) {
7066 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
7067 }
7068 return Register();
7069}
7070
7071Register fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7072 if (RetVT.SimpleTy != MVT::v16i8)
7073 return Register();
7074 if ((Subtarget->hasMVEIntegerOps())) {
7075 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
7076 }
7077 if ((Subtarget->hasNEON())) {
7078 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
7079 }
7080 return Register();
7081}
7082
7083Register fastEmit_ISD_XOR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7084 if (RetVT.SimpleTy != MVT::v4i16)
7085 return Register();
7086 if ((Subtarget->hasNEON())) {
7087 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
7088 }
7089 return Register();
7090}
7091
7092Register fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7093 if (RetVT.SimpleTy != MVT::v8i16)
7094 return Register();
7095 if ((Subtarget->hasMVEIntegerOps())) {
7096 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
7097 }
7098 if ((Subtarget->hasNEON())) {
7099 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
7100 }
7101 return Register();
7102}
7103
7104Register fastEmit_ISD_XOR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7105 if (RetVT.SimpleTy != MVT::v2i32)
7106 return Register();
7107 if ((Subtarget->hasNEON())) {
7108 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
7109 }
7110 return Register();
7111}
7112
7113Register fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7114 if (RetVT.SimpleTy != MVT::v4i32)
7115 return Register();
7116 if ((Subtarget->hasMVEIntegerOps())) {
7117 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
7118 }
7119 if ((Subtarget->hasNEON())) {
7120 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
7121 }
7122 return Register();
7123}
7124
7125Register fastEmit_ISD_XOR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7126 if (RetVT.SimpleTy != MVT::v1i64)
7127 return Register();
7128 if ((Subtarget->hasNEON())) {
7129 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
7130 }
7131 return Register();
7132}
7133
7134Register fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7135 if (RetVT.SimpleTy != MVT::v2i64)
7136 return Register();
7137 if ((Subtarget->hasMVEIntegerOps())) {
7138 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
7139 }
7140 if ((Subtarget->hasNEON())) {
7141 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
7142 }
7143 return Register();
7144}
7145
7146Register fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7147 switch (VT.SimpleTy) {
7148 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
7149 case MVT::v8i8: return fastEmit_ISD_XOR_MVT_v8i8_rr(RetVT, Op0, Op1);
7150 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
7151 case MVT::v4i16: return fastEmit_ISD_XOR_MVT_v4i16_rr(RetVT, Op0, Op1);
7152 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
7153 case MVT::v2i32: return fastEmit_ISD_XOR_MVT_v2i32_rr(RetVT, Op0, Op1);
7154 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
7155 case MVT::v1i64: return fastEmit_ISD_XOR_MVT_v1i64_rr(RetVT, Op0, Op1);
7156 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
7157 default: return Register();
7158 }
7159}
7160
7161// Top-level FastEmit function.
7162
7163Register fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, Register Op1) override {
7164 switch (Opcode) {
7165 case ARMISD::QADD16b: return fastEmit_ARMISD_QADD16b_rr(VT, RetVT, Op0, Op1);
7166 case ARMISD::QADD8b: return fastEmit_ARMISD_QADD8b_rr(VT, RetVT, Op0, Op1);
7167 case ARMISD::QSUB16b: return fastEmit_ARMISD_QSUB16b_rr(VT, RetVT, Op0, Op1);
7168 case ARMISD::QSUB8b: return fastEmit_ARMISD_QSUB8b_rr(VT, RetVT, Op0, Op1);
7169 case ARMISD::SMULWB: return fastEmit_ARMISD_SMULWB_rr(VT, RetVT, Op0, Op1);
7170 case ARMISD::SMULWT: return fastEmit_ARMISD_SMULWT_rr(VT, RetVT, Op0, Op1);
7171 case ARMISD::UQADD16b: return fastEmit_ARMISD_UQADD16b_rr(VT, RetVT, Op0, Op1);
7172 case ARMISD::UQADD8b: return fastEmit_ARMISD_UQADD8b_rr(VT, RetVT, Op0, Op1);
7173 case ARMISD::UQSUB16b: return fastEmit_ARMISD_UQSUB16b_rr(VT, RetVT, Op0, Op1);
7174 case ARMISD::UQSUB8b: return fastEmit_ARMISD_UQSUB8b_rr(VT, RetVT, Op0, Op1);
7175 case ARMISD::VMLAVs: return fastEmit_ARMISD_VMLAVs_rr(VT, RetVT, Op0, Op1);
7176 case ARMISD::VMLAVu: return fastEmit_ARMISD_VMLAVu_rr(VT, RetVT, Op0, Op1);
7177 case ARMISD::VMOVDRR: return fastEmit_ARMISD_VMOVDRR_rr(VT, RetVT, Op0, Op1);
7178 case ARMISD::VMULLs: return fastEmit_ARMISD_VMULLs_rr(VT, RetVT, Op0, Op1);
7179 case ARMISD::VMULLu: return fastEmit_ARMISD_VMULLu_rr(VT, RetVT, Op0, Op1);
7180 case ARMISD::VQDMULH: return fastEmit_ARMISD_VQDMULH_rr(VT, RetVT, Op0, Op1);
7181 case ARMISD::VSHLs: return fastEmit_ARMISD_VSHLs_rr(VT, RetVT, Op0, Op1);
7182 case ARMISD::VSHLu: return fastEmit_ARMISD_VSHLu_rr(VT, RetVT, Op0, Op1);
7183 case ARMISD::VTBL1: return fastEmit_ARMISD_VTBL1_rr(VT, RetVT, Op0, Op1);
7184 case ARMISD::VTST: return fastEmit_ARMISD_VTST_rr(VT, RetVT, Op0, Op1);
7185 case ISD::ABDS: return fastEmit_ISD_ABDS_rr(VT, RetVT, Op0, Op1);
7186 case ISD::ABDU: return fastEmit_ISD_ABDU_rr(VT, RetVT, Op0, Op1);
7187 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
7188 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
7189 case ISD::AVGCEILS: return fastEmit_ISD_AVGCEILS_rr(VT, RetVT, Op0, Op1);
7190 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
7191 case ISD::AVGFLOORS: return fastEmit_ISD_AVGFLOORS_rr(VT, RetVT, Op0, Op1);
7192 case ISD::AVGFLOORU: return fastEmit_ISD_AVGFLOORU_rr(VT, RetVT, Op0, Op1);
7193 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
7194 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
7195 case ISD::FMAXIMUM: return fastEmit_ISD_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
7196 case ISD::FMAXNUM: return fastEmit_ISD_FMAXNUM_rr(VT, RetVT, Op0, Op1);
7197 case ISD::FMINIMUM: return fastEmit_ISD_FMINIMUM_rr(VT, RetVT, Op0, Op1);
7198 case ISD::FMINNUM: return fastEmit_ISD_FMINNUM_rr(VT, RetVT, Op0, Op1);
7199 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
7200 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
7201 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
7202 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
7203 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
7204 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
7205 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
7206 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
7207 case ISD::SDIV: return fastEmit_ISD_SDIV_rr(VT, RetVT, Op0, Op1);
7208 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
7209 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
7210 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
7211 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
7212 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
7213 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
7214 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1);
7215 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1);
7216 case ISD::STRICT_FMAXNUM: return fastEmit_ISD_STRICT_FMAXNUM_rr(VT, RetVT, Op0, Op1);
7217 case ISD::STRICT_FMINNUM: return fastEmit_ISD_STRICT_FMINNUM_rr(VT, RetVT, Op0, Op1);
7218 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1);
7219 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1);
7220 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
7221 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
7222 case ISD::UDIV: return fastEmit_ISD_UDIV_rr(VT, RetVT, Op0, Op1);
7223 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
7224 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
7225 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
7226 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
7227 default: return Register();
7228 }
7229}
7230
7231// FastEmit functions for ARMISD::PIC_ADD.
7232
7233Register fastEmit_ARMISD_PIC_ADD_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7234 if (RetVT.SimpleTy != MVT::i32)
7235 return Register();
7236 if ((Subtarget->isThumb())) {
7237 return fastEmitInst_ri(MachineInstOpcode: ARM::tPICADD, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7238 }
7239 if ((!Subtarget->isThumb())) {
7240 return fastEmitInst_ri(MachineInstOpcode: ARM::PICADD, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7241 }
7242 return Register();
7243}
7244
7245Register fastEmit_ARMISD_PIC_ADD_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7246 switch (VT.SimpleTy) {
7247 case MVT::i32: return fastEmit_ARMISD_PIC_ADD_MVT_i32_ri(RetVT, Op0, imm1);
7248 default: return Register();
7249 }
7250}
7251
7252// FastEmit functions for ARMISD::VDUPLANE.
7253
7254Register fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7255 if (RetVT.SimpleTy != MVT::v8i8)
7256 return Register();
7257 if ((Subtarget->hasNEON())) {
7258 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN8d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7259 }
7260 return Register();
7261}
7262
7263Register fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7264 if (RetVT.SimpleTy != MVT::v4i16)
7265 return Register();
7266 if ((Subtarget->hasNEON())) {
7267 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7268 }
7269 return Register();
7270}
7271
7272Register fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7273 if (RetVT.SimpleTy != MVT::v2i32)
7274 return Register();
7275 if ((Subtarget->hasNEON())) {
7276 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7277 }
7278 return Register();
7279}
7280
7281Register fastEmit_ARMISD_VDUPLANE_MVT_v4f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7282 if (RetVT.SimpleTy != MVT::v4f16)
7283 return Register();
7284 if ((Subtarget->hasNEON())) {
7285 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7286 }
7287 return Register();
7288}
7289
7290Register fastEmit_ARMISD_VDUPLANE_MVT_v4bf16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7291 if (RetVT.SimpleTy != MVT::v4bf16)
7292 return Register();
7293 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
7294 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7295 }
7296 return Register();
7297}
7298
7299Register fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v2f32_ri(Register Op0, uint64_t imm1) {
7300 if ((Subtarget->hasNEON())) {
7301 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7302 }
7303 return Register();
7304}
7305
7306Register fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v4f32_ri(Register Op0, uint64_t imm1) {
7307 if ((Subtarget->hasNEON())) {
7308 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7309 }
7310 return Register();
7311}
7312
7313Register fastEmit_ARMISD_VDUPLANE_MVT_v2f32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7314switch (RetVT.SimpleTy) {
7315 case MVT::v2f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v2f32_ri(Op0, imm1);
7316 case MVT::v4f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v4f32_ri(Op0, imm1);
7317 default: return Register();
7318}
7319}
7320
7321Register fastEmit_ARMISD_VDUPLANE_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7322 switch (VT.SimpleTy) {
7323 case MVT::v8i8: return fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri(RetVT, Op0, imm1);
7324 case MVT::v4i16: return fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri(RetVT, Op0, imm1);
7325 case MVT::v2i32: return fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri(RetVT, Op0, imm1);
7326 case MVT::v4f16: return fastEmit_ARMISD_VDUPLANE_MVT_v4f16_ri(RetVT, Op0, imm1);
7327 case MVT::v4bf16: return fastEmit_ARMISD_VDUPLANE_MVT_v4bf16_ri(RetVT, Op0, imm1);
7328 case MVT::v2f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_ri(RetVT, Op0, imm1);
7329 default: return Register();
7330 }
7331}
7332
7333// FastEmit functions for ARMISD::VGETLANEs.
7334
7335Register fastEmit_ARMISD_VGETLANEs_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7336 if (RetVT.SimpleTy != MVT::i32)
7337 return Register();
7338 if ((Subtarget->hasNEON())) {
7339 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNs8, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7340 }
7341 return Register();
7342}
7343
7344Register fastEmit_ARMISD_VGETLANEs_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7345 if (RetVT.SimpleTy != MVT::i32)
7346 return Register();
7347 if ((Subtarget->hasMVEIntegerOps())) {
7348 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_s8, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7349 }
7350 return Register();
7351}
7352
7353Register fastEmit_ARMISD_VGETLANEs_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7354 if (RetVT.SimpleTy != MVT::i32)
7355 return Register();
7356 if ((Subtarget->hasNEON())) {
7357 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNs16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7358 }
7359 return Register();
7360}
7361
7362Register fastEmit_ARMISD_VGETLANEs_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7363 if (RetVT.SimpleTy != MVT::i32)
7364 return Register();
7365 if ((Subtarget->hasMVEIntegerOps())) {
7366 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_s16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7367 }
7368 return Register();
7369}
7370
7371Register fastEmit_ARMISD_VGETLANEs_MVT_v8f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7372 if (RetVT.SimpleTy != MVT::i32)
7373 return Register();
7374 if ((Subtarget->hasMVEIntegerOps())) {
7375 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_s16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7376 }
7377 return Register();
7378}
7379
7380Register fastEmit_ARMISD_VGETLANEs_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7381 switch (VT.SimpleTy) {
7382 case MVT::v8i8: return fastEmit_ARMISD_VGETLANEs_MVT_v8i8_ri(RetVT, Op0, imm1);
7383 case MVT::v16i8: return fastEmit_ARMISD_VGETLANEs_MVT_v16i8_ri(RetVT, Op0, imm1);
7384 case MVT::v4i16: return fastEmit_ARMISD_VGETLANEs_MVT_v4i16_ri(RetVT, Op0, imm1);
7385 case MVT::v8i16: return fastEmit_ARMISD_VGETLANEs_MVT_v8i16_ri(RetVT, Op0, imm1);
7386 case MVT::v8f16: return fastEmit_ARMISD_VGETLANEs_MVT_v8f16_ri(RetVT, Op0, imm1);
7387 default: return Register();
7388 }
7389}
7390
7391// FastEmit functions for ARMISD::VGETLANEu.
7392
7393Register fastEmit_ARMISD_VGETLANEu_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7394 if (RetVT.SimpleTy != MVT::i32)
7395 return Register();
7396 if ((Subtarget->hasNEON())) {
7397 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu8, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7398 }
7399 return Register();
7400}
7401
7402Register fastEmit_ARMISD_VGETLANEu_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7403 if (RetVT.SimpleTy != MVT::i32)
7404 return Register();
7405 if ((Subtarget->hasMVEIntegerOps())) {
7406 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_u8, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7407 }
7408 return Register();
7409}
7410
7411Register fastEmit_ARMISD_VGETLANEu_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7412 if (RetVT.SimpleTy != MVT::i32)
7413 return Register();
7414 if ((Subtarget->hasNEON())) {
7415 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7416 }
7417 return Register();
7418}
7419
7420Register fastEmit_ARMISD_VGETLANEu_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7421 if (RetVT.SimpleTy != MVT::i32)
7422 return Register();
7423 if ((Subtarget->hasMVEIntegerOps())) {
7424 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_u16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7425 }
7426 return Register();
7427}
7428
7429Register fastEmit_ARMISD_VGETLANEu_MVT_v4f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7430 if (RetVT.SimpleTy != MVT::i32)
7431 return Register();
7432 if ((Subtarget->hasNEON())) {
7433 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7434 }
7435 return Register();
7436}
7437
7438Register fastEmit_ARMISD_VGETLANEu_MVT_v8f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7439 if (RetVT.SimpleTy != MVT::i32)
7440 return Register();
7441 if ((Subtarget->hasMVEIntegerOps())) {
7442 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_u16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7443 }
7444 return Register();
7445}
7446
7447Register fastEmit_ARMISD_VGETLANEu_MVT_v4bf16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7448 if (RetVT.SimpleTy != MVT::i32)
7449 return Register();
7450 if ((Subtarget->hasNEON())) {
7451 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7452 }
7453 return Register();
7454}
7455
7456Register fastEmit_ARMISD_VGETLANEu_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7457 switch (VT.SimpleTy) {
7458 case MVT::v8i8: return fastEmit_ARMISD_VGETLANEu_MVT_v8i8_ri(RetVT, Op0, imm1);
7459 case MVT::v16i8: return fastEmit_ARMISD_VGETLANEu_MVT_v16i8_ri(RetVT, Op0, imm1);
7460 case MVT::v4i16: return fastEmit_ARMISD_VGETLANEu_MVT_v4i16_ri(RetVT, Op0, imm1);
7461 case MVT::v8i16: return fastEmit_ARMISD_VGETLANEu_MVT_v8i16_ri(RetVT, Op0, imm1);
7462 case MVT::v4f16: return fastEmit_ARMISD_VGETLANEu_MVT_v4f16_ri(RetVT, Op0, imm1);
7463 case MVT::v8f16: return fastEmit_ARMISD_VGETLANEu_MVT_v8f16_ri(RetVT, Op0, imm1);
7464 case MVT::v4bf16: return fastEmit_ARMISD_VGETLANEu_MVT_v4bf16_ri(RetVT, Op0, imm1);
7465 default: return Register();
7466 }
7467}
7468
7469// FastEmit functions for ARMISD::VQSHLsIMM.
7470
7471Register fastEmit_ARMISD_VQSHLsIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7472 if (RetVT.SimpleTy != MVT::v8i8)
7473 return Register();
7474 if ((Subtarget->hasNEON())) {
7475 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7476 }
7477 return Register();
7478}
7479
7480Register fastEmit_ARMISD_VQSHLsIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7481 if (RetVT.SimpleTy != MVT::v16i8)
7482 return Register();
7483 if ((Subtarget->hasNEON())) {
7484 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7485 }
7486 return Register();
7487}
7488
7489Register fastEmit_ARMISD_VQSHLsIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7490 if (RetVT.SimpleTy != MVT::v4i16)
7491 return Register();
7492 if ((Subtarget->hasNEON())) {
7493 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7494 }
7495 return Register();
7496}
7497
7498Register fastEmit_ARMISD_VQSHLsIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7499 if (RetVT.SimpleTy != MVT::v8i16)
7500 return Register();
7501 if ((Subtarget->hasNEON())) {
7502 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7503 }
7504 return Register();
7505}
7506
7507Register fastEmit_ARMISD_VQSHLsIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7508 if (RetVT.SimpleTy != MVT::v2i32)
7509 return Register();
7510 if ((Subtarget->hasNEON())) {
7511 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7512 }
7513 return Register();
7514}
7515
7516Register fastEmit_ARMISD_VQSHLsIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7517 if (RetVT.SimpleTy != MVT::v4i32)
7518 return Register();
7519 if ((Subtarget->hasNEON())) {
7520 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7521 }
7522 return Register();
7523}
7524
7525Register fastEmit_ARMISD_VQSHLsIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7526 if (RetVT.SimpleTy != MVT::v1i64)
7527 return Register();
7528 if ((Subtarget->hasNEON())) {
7529 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7530 }
7531 return Register();
7532}
7533
7534Register fastEmit_ARMISD_VQSHLsIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7535 if (RetVT.SimpleTy != MVT::v2i64)
7536 return Register();
7537 if ((Subtarget->hasNEON())) {
7538 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7539 }
7540 return Register();
7541}
7542
7543Register fastEmit_ARMISD_VQSHLsIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7544 switch (VT.SimpleTy) {
7545 case MVT::v8i8: return fastEmit_ARMISD_VQSHLsIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7546 case MVT::v16i8: return fastEmit_ARMISD_VQSHLsIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7547 case MVT::v4i16: return fastEmit_ARMISD_VQSHLsIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7548 case MVT::v8i16: return fastEmit_ARMISD_VQSHLsIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7549 case MVT::v2i32: return fastEmit_ARMISD_VQSHLsIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7550 case MVT::v4i32: return fastEmit_ARMISD_VQSHLsIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7551 case MVT::v1i64: return fastEmit_ARMISD_VQSHLsIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7552 case MVT::v2i64: return fastEmit_ARMISD_VQSHLsIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7553 default: return Register();
7554 }
7555}
7556
7557// FastEmit functions for ARMISD::VQSHLsuIMM.
7558
7559Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7560 if (RetVT.SimpleTy != MVT::v8i8)
7561 return Register();
7562 if ((Subtarget->hasNEON())) {
7563 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7564 }
7565 return Register();
7566}
7567
7568Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7569 if (RetVT.SimpleTy != MVT::v16i8)
7570 return Register();
7571 if ((Subtarget->hasNEON())) {
7572 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7573 }
7574 return Register();
7575}
7576
7577Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7578 if (RetVT.SimpleTy != MVT::v4i16)
7579 return Register();
7580 if ((Subtarget->hasNEON())) {
7581 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7582 }
7583 return Register();
7584}
7585
7586Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7587 if (RetVT.SimpleTy != MVT::v8i16)
7588 return Register();
7589 if ((Subtarget->hasNEON())) {
7590 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7591 }
7592 return Register();
7593}
7594
7595Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7596 if (RetVT.SimpleTy != MVT::v2i32)
7597 return Register();
7598 if ((Subtarget->hasNEON())) {
7599 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7600 }
7601 return Register();
7602}
7603
7604Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7605 if (RetVT.SimpleTy != MVT::v4i32)
7606 return Register();
7607 if ((Subtarget->hasNEON())) {
7608 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7609 }
7610 return Register();
7611}
7612
7613Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7614 if (RetVT.SimpleTy != MVT::v1i64)
7615 return Register();
7616 if ((Subtarget->hasNEON())) {
7617 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7618 }
7619 return Register();
7620}
7621
7622Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7623 if (RetVT.SimpleTy != MVT::v2i64)
7624 return Register();
7625 if ((Subtarget->hasNEON())) {
7626 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7627 }
7628 return Register();
7629}
7630
7631Register fastEmit_ARMISD_VQSHLsuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7632 switch (VT.SimpleTy) {
7633 case MVT::v8i8: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7634 case MVT::v16i8: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7635 case MVT::v4i16: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7636 case MVT::v8i16: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7637 case MVT::v2i32: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7638 case MVT::v4i32: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7639 case MVT::v1i64: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7640 case MVT::v2i64: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7641 default: return Register();
7642 }
7643}
7644
7645// FastEmit functions for ARMISD::VQSHLuIMM.
7646
7647Register fastEmit_ARMISD_VQSHLuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7648 if (RetVT.SimpleTy != MVT::v8i8)
7649 return Register();
7650 if ((Subtarget->hasNEON())) {
7651 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7652 }
7653 return Register();
7654}
7655
7656Register fastEmit_ARMISD_VQSHLuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7657 if (RetVT.SimpleTy != MVT::v16i8)
7658 return Register();
7659 if ((Subtarget->hasNEON())) {
7660 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7661 }
7662 return Register();
7663}
7664
7665Register fastEmit_ARMISD_VQSHLuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7666 if (RetVT.SimpleTy != MVT::v4i16)
7667 return Register();
7668 if ((Subtarget->hasNEON())) {
7669 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7670 }
7671 return Register();
7672}
7673
7674Register fastEmit_ARMISD_VQSHLuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7675 if (RetVT.SimpleTy != MVT::v8i16)
7676 return Register();
7677 if ((Subtarget->hasNEON())) {
7678 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7679 }
7680 return Register();
7681}
7682
7683Register fastEmit_ARMISD_VQSHLuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7684 if (RetVT.SimpleTy != MVT::v2i32)
7685 return Register();
7686 if ((Subtarget->hasNEON())) {
7687 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7688 }
7689 return Register();
7690}
7691
7692Register fastEmit_ARMISD_VQSHLuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7693 if (RetVT.SimpleTy != MVT::v4i32)
7694 return Register();
7695 if ((Subtarget->hasNEON())) {
7696 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7697 }
7698 return Register();
7699}
7700
7701Register fastEmit_ARMISD_VQSHLuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7702 if (RetVT.SimpleTy != MVT::v1i64)
7703 return Register();
7704 if ((Subtarget->hasNEON())) {
7705 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7706 }
7707 return Register();
7708}
7709
7710Register fastEmit_ARMISD_VQSHLuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7711 if (RetVT.SimpleTy != MVT::v2i64)
7712 return Register();
7713 if ((Subtarget->hasNEON())) {
7714 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7715 }
7716 return Register();
7717}
7718
7719Register fastEmit_ARMISD_VQSHLuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7720 switch (VT.SimpleTy) {
7721 case MVT::v8i8: return fastEmit_ARMISD_VQSHLuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7722 case MVT::v16i8: return fastEmit_ARMISD_VQSHLuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7723 case MVT::v4i16: return fastEmit_ARMISD_VQSHLuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7724 case MVT::v8i16: return fastEmit_ARMISD_VQSHLuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7725 case MVT::v2i32: return fastEmit_ARMISD_VQSHLuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7726 case MVT::v4i32: return fastEmit_ARMISD_VQSHLuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7727 case MVT::v1i64: return fastEmit_ARMISD_VQSHLuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7728 case MVT::v2i64: return fastEmit_ARMISD_VQSHLuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7729 default: return Register();
7730 }
7731}
7732
7733// FastEmit functions for ARMISD::VRSHRsIMM.
7734
7735Register fastEmit_ARMISD_VRSHRsIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7736 if (RetVT.SimpleTy != MVT::v8i8)
7737 return Register();
7738 if ((Subtarget->hasNEON())) {
7739 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7740 }
7741 return Register();
7742}
7743
7744Register fastEmit_ARMISD_VRSHRsIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7745 if (RetVT.SimpleTy != MVT::v16i8)
7746 return Register();
7747 if ((Subtarget->hasNEON())) {
7748 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7749 }
7750 return Register();
7751}
7752
7753Register fastEmit_ARMISD_VRSHRsIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7754 if (RetVT.SimpleTy != MVT::v4i16)
7755 return Register();
7756 if ((Subtarget->hasNEON())) {
7757 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7758 }
7759 return Register();
7760}
7761
7762Register fastEmit_ARMISD_VRSHRsIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7763 if (RetVT.SimpleTy != MVT::v8i16)
7764 return Register();
7765 if ((Subtarget->hasNEON())) {
7766 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7767 }
7768 return Register();
7769}
7770
7771Register fastEmit_ARMISD_VRSHRsIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7772 if (RetVT.SimpleTy != MVT::v2i32)
7773 return Register();
7774 if ((Subtarget->hasNEON())) {
7775 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7776 }
7777 return Register();
7778}
7779
7780Register fastEmit_ARMISD_VRSHRsIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7781 if (RetVT.SimpleTy != MVT::v4i32)
7782 return Register();
7783 if ((Subtarget->hasNEON())) {
7784 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7785 }
7786 return Register();
7787}
7788
7789Register fastEmit_ARMISD_VRSHRsIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7790 if (RetVT.SimpleTy != MVT::v1i64)
7791 return Register();
7792 if ((Subtarget->hasNEON())) {
7793 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7794 }
7795 return Register();
7796}
7797
7798Register fastEmit_ARMISD_VRSHRsIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7799 if (RetVT.SimpleTy != MVT::v2i64)
7800 return Register();
7801 if ((Subtarget->hasNEON())) {
7802 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7803 }
7804 return Register();
7805}
7806
7807Register fastEmit_ARMISD_VRSHRsIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7808 switch (VT.SimpleTy) {
7809 case MVT::v8i8: return fastEmit_ARMISD_VRSHRsIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7810 case MVT::v16i8: return fastEmit_ARMISD_VRSHRsIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7811 case MVT::v4i16: return fastEmit_ARMISD_VRSHRsIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7812 case MVT::v8i16: return fastEmit_ARMISD_VRSHRsIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7813 case MVT::v2i32: return fastEmit_ARMISD_VRSHRsIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7814 case MVT::v4i32: return fastEmit_ARMISD_VRSHRsIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7815 case MVT::v1i64: return fastEmit_ARMISD_VRSHRsIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7816 case MVT::v2i64: return fastEmit_ARMISD_VRSHRsIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7817 default: return Register();
7818 }
7819}
7820
7821// FastEmit functions for ARMISD::VRSHRuIMM.
7822
7823Register fastEmit_ARMISD_VRSHRuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7824 if (RetVT.SimpleTy != MVT::v8i8)
7825 return Register();
7826 if ((Subtarget->hasNEON())) {
7827 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7828 }
7829 return Register();
7830}
7831
7832Register fastEmit_ARMISD_VRSHRuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7833 if (RetVT.SimpleTy != MVT::v16i8)
7834 return Register();
7835 if ((Subtarget->hasNEON())) {
7836 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7837 }
7838 return Register();
7839}
7840
7841Register fastEmit_ARMISD_VRSHRuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7842 if (RetVT.SimpleTy != MVT::v4i16)
7843 return Register();
7844 if ((Subtarget->hasNEON())) {
7845 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7846 }
7847 return Register();
7848}
7849
7850Register fastEmit_ARMISD_VRSHRuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7851 if (RetVT.SimpleTy != MVT::v8i16)
7852 return Register();
7853 if ((Subtarget->hasNEON())) {
7854 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7855 }
7856 return Register();
7857}
7858
7859Register fastEmit_ARMISD_VRSHRuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7860 if (RetVT.SimpleTy != MVT::v2i32)
7861 return Register();
7862 if ((Subtarget->hasNEON())) {
7863 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7864 }
7865 return Register();
7866}
7867
7868Register fastEmit_ARMISD_VRSHRuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7869 if (RetVT.SimpleTy != MVT::v4i32)
7870 return Register();
7871 if ((Subtarget->hasNEON())) {
7872 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7873 }
7874 return Register();
7875}
7876
7877Register fastEmit_ARMISD_VRSHRuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7878 if (RetVT.SimpleTy != MVT::v1i64)
7879 return Register();
7880 if ((Subtarget->hasNEON())) {
7881 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7882 }
7883 return Register();
7884}
7885
7886Register fastEmit_ARMISD_VRSHRuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7887 if (RetVT.SimpleTy != MVT::v2i64)
7888 return Register();
7889 if ((Subtarget->hasNEON())) {
7890 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7891 }
7892 return Register();
7893}
7894
7895Register fastEmit_ARMISD_VRSHRuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7896 switch (VT.SimpleTy) {
7897 case MVT::v8i8: return fastEmit_ARMISD_VRSHRuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7898 case MVT::v16i8: return fastEmit_ARMISD_VRSHRuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7899 case MVT::v4i16: return fastEmit_ARMISD_VRSHRuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7900 case MVT::v8i16: return fastEmit_ARMISD_VRSHRuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7901 case MVT::v2i32: return fastEmit_ARMISD_VRSHRuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7902 case MVT::v4i32: return fastEmit_ARMISD_VRSHRuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7903 case MVT::v1i64: return fastEmit_ARMISD_VRSHRuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7904 case MVT::v2i64: return fastEmit_ARMISD_VRSHRuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7905 default: return Register();
7906 }
7907}
7908
7909// FastEmit functions for ARMISD::VSHLIMM.
7910
7911Register fastEmit_ARMISD_VSHLIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7912 if (RetVT.SimpleTy != MVT::v8i8)
7913 return Register();
7914 if ((Subtarget->hasNEON())) {
7915 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7916 }
7917 return Register();
7918}
7919
7920Register fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7921 if (RetVT.SimpleTy != MVT::v16i8)
7922 return Register();
7923 if ((Subtarget->hasNEON())) {
7924 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7925 }
7926 return Register();
7927}
7928
7929Register fastEmit_ARMISD_VSHLIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7930 if (RetVT.SimpleTy != MVT::v4i16)
7931 return Register();
7932 if ((Subtarget->hasNEON())) {
7933 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7934 }
7935 return Register();
7936}
7937
7938Register fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7939 if (RetVT.SimpleTy != MVT::v8i16)
7940 return Register();
7941 if ((Subtarget->hasNEON())) {
7942 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7943 }
7944 return Register();
7945}
7946
7947Register fastEmit_ARMISD_VSHLIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7948 if (RetVT.SimpleTy != MVT::v2i32)
7949 return Register();
7950 if ((Subtarget->hasNEON())) {
7951 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7952 }
7953 return Register();
7954}
7955
7956Register fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7957 if (RetVT.SimpleTy != MVT::v4i32)
7958 return Register();
7959 if ((Subtarget->hasNEON())) {
7960 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7961 }
7962 return Register();
7963}
7964
7965Register fastEmit_ARMISD_VSHLIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7966 if (RetVT.SimpleTy != MVT::v1i64)
7967 return Register();
7968 if ((Subtarget->hasNEON())) {
7969 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7970 }
7971 return Register();
7972}
7973
7974Register fastEmit_ARMISD_VSHLIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7975 if (RetVT.SimpleTy != MVT::v2i64)
7976 return Register();
7977 if ((Subtarget->hasNEON())) {
7978 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7979 }
7980 return Register();
7981}
7982
7983Register fastEmit_ARMISD_VSHLIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7984 switch (VT.SimpleTy) {
7985 case MVT::v8i8: return fastEmit_ARMISD_VSHLIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7986 case MVT::v16i8: return fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7987 case MVT::v4i16: return fastEmit_ARMISD_VSHLIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7988 case MVT::v8i16: return fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7989 case MVT::v2i32: return fastEmit_ARMISD_VSHLIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7990 case MVT::v4i32: return fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7991 case MVT::v1i64: return fastEmit_ARMISD_VSHLIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7992 case MVT::v2i64: return fastEmit_ARMISD_VSHLIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7993 default: return Register();
7994 }
7995}
7996
7997// FastEmit functions for ARMISD::VSHRsIMM.
7998
7999Register fastEmit_ARMISD_VSHRsIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8000 if (RetVT.SimpleTy != MVT::v8i8)
8001 return Register();
8002 if ((Subtarget->hasNEON())) {
8003 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8004 }
8005 return Register();
8006}
8007
8008Register fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8009 if (RetVT.SimpleTy != MVT::v16i8)
8010 return Register();
8011 if ((Subtarget->hasNEON())) {
8012 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8013 }
8014 return Register();
8015}
8016
8017Register fastEmit_ARMISD_VSHRsIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8018 if (RetVT.SimpleTy != MVT::v4i16)
8019 return Register();
8020 if ((Subtarget->hasNEON())) {
8021 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8022 }
8023 return Register();
8024}
8025
8026Register fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8027 if (RetVT.SimpleTy != MVT::v8i16)
8028 return Register();
8029 if ((Subtarget->hasNEON())) {
8030 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8031 }
8032 return Register();
8033}
8034
8035Register fastEmit_ARMISD_VSHRsIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8036 if (RetVT.SimpleTy != MVT::v2i32)
8037 return Register();
8038 if ((Subtarget->hasNEON())) {
8039 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8040 }
8041 return Register();
8042}
8043
8044Register fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8045 if (RetVT.SimpleTy != MVT::v4i32)
8046 return Register();
8047 if ((Subtarget->hasNEON())) {
8048 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8049 }
8050 return Register();
8051}
8052
8053Register fastEmit_ARMISD_VSHRsIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8054 if (RetVT.SimpleTy != MVT::v1i64)
8055 return Register();
8056 if ((Subtarget->hasNEON())) {
8057 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8058 }
8059 return Register();
8060}
8061
8062Register fastEmit_ARMISD_VSHRsIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8063 if (RetVT.SimpleTy != MVT::v2i64)
8064 return Register();
8065 if ((Subtarget->hasNEON())) {
8066 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8067 }
8068 return Register();
8069}
8070
8071Register fastEmit_ARMISD_VSHRsIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8072 switch (VT.SimpleTy) {
8073 case MVT::v8i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8074 case MVT::v16i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8075 case MVT::v4i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8076 case MVT::v8i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8077 case MVT::v2i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8078 case MVT::v4i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8079 case MVT::v1i64: return fastEmit_ARMISD_VSHRsIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8080 case MVT::v2i64: return fastEmit_ARMISD_VSHRsIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8081 default: return Register();
8082 }
8083}
8084
8085// FastEmit functions for ARMISD::VSHRuIMM.
8086
8087Register fastEmit_ARMISD_VSHRuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8088 if (RetVT.SimpleTy != MVT::v8i8)
8089 return Register();
8090 if ((Subtarget->hasNEON())) {
8091 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8092 }
8093 return Register();
8094}
8095
8096Register fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8097 if (RetVT.SimpleTy != MVT::v16i8)
8098 return Register();
8099 if ((Subtarget->hasNEON())) {
8100 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8101 }
8102 return Register();
8103}
8104
8105Register fastEmit_ARMISD_VSHRuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8106 if (RetVT.SimpleTy != MVT::v4i16)
8107 return Register();
8108 if ((Subtarget->hasNEON())) {
8109 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8110 }
8111 return Register();
8112}
8113
8114Register fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8115 if (RetVT.SimpleTy != MVT::v8i16)
8116 return Register();
8117 if ((Subtarget->hasNEON())) {
8118 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8119 }
8120 return Register();
8121}
8122
8123Register fastEmit_ARMISD_VSHRuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8124 if (RetVT.SimpleTy != MVT::v2i32)
8125 return Register();
8126 if ((Subtarget->hasNEON())) {
8127 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8128 }
8129 return Register();
8130}
8131
8132Register fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8133 if (RetVT.SimpleTy != MVT::v4i32)
8134 return Register();
8135 if ((Subtarget->hasNEON())) {
8136 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8137 }
8138 return Register();
8139}
8140
8141Register fastEmit_ARMISD_VSHRuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8142 if (RetVT.SimpleTy != MVT::v1i64)
8143 return Register();
8144 if ((Subtarget->hasNEON())) {
8145 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8146 }
8147 return Register();
8148}
8149
8150Register fastEmit_ARMISD_VSHRuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8151 if (RetVT.SimpleTy != MVT::v2i64)
8152 return Register();
8153 if ((Subtarget->hasNEON())) {
8154 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8155 }
8156 return Register();
8157}
8158
8159Register fastEmit_ARMISD_VSHRuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8160 switch (VT.SimpleTy) {
8161 case MVT::v8i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8162 case MVT::v16i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8163 case MVT::v4i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8164 case MVT::v8i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8165 case MVT::v2i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8166 case MVT::v4i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8167 case MVT::v1i64: return fastEmit_ARMISD_VSHRuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8168 case MVT::v2i64: return fastEmit_ARMISD_VSHRuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8169 default: return Register();
8170 }
8171}
8172
8173// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
8174
8175Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8176 if (RetVT.SimpleTy != MVT::i32)
8177 return Register();
8178 if ((Subtarget->hasFPRegs()) && (!Subtarget->hasSlowVGETLNi32())) {
8179 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNi32, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8180 }
8181 return Register();
8182}
8183
8184Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8185 switch (VT.SimpleTy) {
8186 case MVT::v2i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i32_ri(RetVT, Op0, imm1);
8187 default: return Register();
8188 }
8189}
8190
8191// FastEmit functions for ISD::SHL.
8192
8193Register fastEmit_ISD_SHL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8194 if (RetVT.SimpleTy != MVT::i32)
8195 return Register();
8196 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
8197 return fastEmitInst_ri(MachineInstOpcode: ARM::tLSLri, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
8198 }
8199 return Register();
8200}
8201
8202Register fastEmit_ISD_SHL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8203 switch (VT.SimpleTy) {
8204 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri(RetVT, Op0, imm1);
8205 default: return Register();
8206 }
8207}
8208
8209// Top-level FastEmit function.
8210
8211Register fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) override {
8212 if (VT == MVT::i32 && Predicate_mod_imm(Imm: imm1))
8213 if (Register Reg = fastEmit_ri_Predicate_mod_imm(VT, RetVT, Opcode, Op0, imm1))
8214 return Reg;
8215
8216 if (VT == MVT::i32 && Predicate_imm0_7(Imm: imm1))
8217 if (Register Reg = fastEmit_ri_Predicate_imm0_7(VT, RetVT, Opcode, Op0, imm1))
8218 return Reg;
8219
8220 if (VT == MVT::i32 && Predicate_imm0_255_expr(Imm: imm1))
8221 if (Register Reg = fastEmit_ri_Predicate_imm0_255_expr(VT, RetVT, Opcode, Op0, imm1))
8222 return Reg;
8223
8224 if (VT == MVT::i32 && Predicate_t2_so_imm(Imm: imm1))
8225 if (Register Reg = fastEmit_ri_Predicate_t2_so_imm(VT, RetVT, Opcode, Op0, imm1))
8226 return Reg;
8227
8228 if (VT == MVT::i32 && Predicate_imm0_4095(Imm: imm1))
8229 if (Register Reg = fastEmit_ri_Predicate_imm0_4095(VT, RetVT, Opcode, Op0, imm1))
8230 return Reg;
8231
8232 if (VT == MVT::i32 && Predicate_imm1_31(Imm: imm1))
8233 if (Register Reg = fastEmit_ri_Predicate_imm1_31(VT, RetVT, Opcode, Op0, imm1))
8234 return Reg;
8235
8236 if (VT == MVT::i32 && Predicate_shr_imm8(Imm: imm1))
8237 if (Register Reg = fastEmit_ri_Predicate_shr_imm8(VT, RetVT, Opcode, Op0, imm1))
8238 return Reg;
8239
8240 if (VT == MVT::i32 && Predicate_shr_imm16(Imm: imm1))
8241 if (Register Reg = fastEmit_ri_Predicate_shr_imm16(VT, RetVT, Opcode, Op0, imm1))
8242 return Reg;
8243
8244 if (VT == MVT::i32 && Predicate_shr_imm32(Imm: imm1))
8245 if (Register Reg = fastEmit_ri_Predicate_shr_imm32(VT, RetVT, Opcode, Op0, imm1))
8246 return Reg;
8247
8248 if (VT == MVT::i32 && Predicate_VectorIndex32(Imm: imm1))
8249 if (Register Reg = fastEmit_ri_Predicate_VectorIndex32(VT, RetVT, Opcode, Op0, imm1))
8250 return Reg;
8251
8252 if (VT == MVT::i32 && Predicate_imm0_31(Imm: imm1))
8253 if (Register Reg = fastEmit_ri_Predicate_imm0_31(VT, RetVT, Opcode, Op0, imm1))
8254 return Reg;
8255
8256 if (VT == MVT::i32 && Predicate_imm0_15(Imm: imm1))
8257 if (Register Reg = fastEmit_ri_Predicate_imm0_15(VT, RetVT, Opcode, Op0, imm1))
8258 return Reg;
8259
8260 switch (Opcode) {
8261 case ARMISD::PIC_ADD: return fastEmit_ARMISD_PIC_ADD_ri(VT, RetVT, Op0, imm1);
8262 case ARMISD::VDUPLANE: return fastEmit_ARMISD_VDUPLANE_ri(VT, RetVT, Op0, imm1);
8263 case ARMISD::VGETLANEs: return fastEmit_ARMISD_VGETLANEs_ri(VT, RetVT, Op0, imm1);
8264 case ARMISD::VGETLANEu: return fastEmit_ARMISD_VGETLANEu_ri(VT, RetVT, Op0, imm1);
8265 case ARMISD::VQSHLsIMM: return fastEmit_ARMISD_VQSHLsIMM_ri(VT, RetVT, Op0, imm1);
8266 case ARMISD::VQSHLsuIMM: return fastEmit_ARMISD_VQSHLsuIMM_ri(VT, RetVT, Op0, imm1);
8267 case ARMISD::VQSHLuIMM: return fastEmit_ARMISD_VQSHLuIMM_ri(VT, RetVT, Op0, imm1);
8268 case ARMISD::VRSHRsIMM: return fastEmit_ARMISD_VRSHRsIMM_ri(VT, RetVT, Op0, imm1);
8269 case ARMISD::VRSHRuIMM: return fastEmit_ARMISD_VRSHRuIMM_ri(VT, RetVT, Op0, imm1);
8270 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri(VT, RetVT, Op0, imm1);
8271 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri(VT, RetVT, Op0, imm1);
8272 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri(VT, RetVT, Op0, imm1);
8273 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(VT, RetVT, Op0, imm1);
8274 case ISD::SHL: return fastEmit_ISD_SHL_ri(VT, RetVT, Op0, imm1);
8275 default: return Register();
8276 }
8277}
8278
8279// FastEmit functions for ISD::ADD.
8280
8281Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8282 if (RetVT.SimpleTy != MVT::i32)
8283 return Register();
8284 if ((!Subtarget->isThumb())) {
8285 return fastEmitInst_ri(MachineInstOpcode: ARM::ADDri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8286 }
8287 return Register();
8288}
8289
8290Register fastEmit_ISD_ADD_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8291 switch (VT.SimpleTy) {
8292 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8293 default: return Register();
8294 }
8295}
8296
8297// FastEmit functions for ISD::AND.
8298
8299Register fastEmit_ISD_AND_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8300 if (RetVT.SimpleTy != MVT::i32)
8301 return Register();
8302 if ((!Subtarget->isThumb())) {
8303 return fastEmitInst_ri(MachineInstOpcode: ARM::ANDri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8304 }
8305 return Register();
8306}
8307
8308Register fastEmit_ISD_AND_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8309 switch (VT.SimpleTy) {
8310 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8311 default: return Register();
8312 }
8313}
8314
8315// FastEmit functions for ISD::OR.
8316
8317Register fastEmit_ISD_OR_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8318 if (RetVT.SimpleTy != MVT::i32)
8319 return Register();
8320 if ((!Subtarget->isThumb())) {
8321 return fastEmitInst_ri(MachineInstOpcode: ARM::ORRri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8322 }
8323 return Register();
8324}
8325
8326Register fastEmit_ISD_OR_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8327 switch (VT.SimpleTy) {
8328 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8329 default: return Register();
8330 }
8331}
8332
8333// FastEmit functions for ISD::SUB.
8334
8335Register fastEmit_ISD_SUB_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8336 if (RetVT.SimpleTy != MVT::i32)
8337 return Register();
8338 if ((!Subtarget->isThumb())) {
8339 return fastEmitInst_ri(MachineInstOpcode: ARM::SUBri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8340 }
8341 return Register();
8342}
8343
8344Register fastEmit_ISD_SUB_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8345 switch (VT.SimpleTy) {
8346 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8347 default: return Register();
8348 }
8349}
8350
8351// FastEmit functions for ISD::XOR.
8352
8353Register fastEmit_ISD_XOR_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8354 if (RetVT.SimpleTy != MVT::i32)
8355 return Register();
8356 if ((!Subtarget->isThumb())) {
8357 return fastEmitInst_ri(MachineInstOpcode: ARM::EORri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8358 }
8359 return Register();
8360}
8361
8362Register fastEmit_ISD_XOR_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8363 switch (VT.SimpleTy) {
8364 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8365 default: return Register();
8366 }
8367}
8368
8369// Top-level FastEmit function.
8370
8371Register fastEmit_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8372 switch (Opcode) {
8373 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8374 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8375 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8376 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8377 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8378 default: return Register();
8379 }
8380}
8381
8382// FastEmit functions for ARMISD::VSHLIMM.
8383
8384Register fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
8385 if (RetVT.SimpleTy != MVT::v16i8)
8386 return Register();
8387 if ((Subtarget->hasMVEIntegerOps())) {
8388 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHL_immi8, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8389 }
8390 return Register();
8391}
8392
8393Register fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8394 switch (VT.SimpleTy) {
8395 case MVT::v16i8: return fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, imm1);
8396 default: return Register();
8397 }
8398}
8399
8400// FastEmit functions for ARMISD::VSHRsIMM.
8401
8402Register fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
8403 if (RetVT.SimpleTy != MVT::v16i8)
8404 return Register();
8405 if ((Subtarget->hasMVEIntegerOps())) {
8406 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_imms8, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8407 }
8408 return Register();
8409}
8410
8411Register fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8412 switch (VT.SimpleTy) {
8413 case MVT::v16i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, imm1);
8414 default: return Register();
8415 }
8416}
8417
8418// FastEmit functions for ARMISD::VSHRuIMM.
8419
8420Register fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
8421 if (RetVT.SimpleTy != MVT::v16i8)
8422 return Register();
8423 if ((Subtarget->hasMVEIntegerOps())) {
8424 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_immu8, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8425 }
8426 return Register();
8427}
8428
8429Register fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8430 switch (VT.SimpleTy) {
8431 case MVT::v16i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, imm1);
8432 default: return Register();
8433 }
8434}
8435
8436// FastEmit functions for ISD::ADD.
8437
8438Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
8439 if (RetVT.SimpleTy != MVT::i32)
8440 return Register();
8441 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
8442 return fastEmitInst_ri(MachineInstOpcode: ARM::tADDi3, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
8443 }
8444 return Register();
8445}
8446
8447Register fastEmit_ISD_ADD_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8448 switch (VT.SimpleTy) {
8449 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_7(RetVT, Op0, imm1);
8450 default: return Register();
8451 }
8452}
8453
8454// Top-level FastEmit function.
8455
8456Register fastEmit_ri_Predicate_imm0_7(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8457 switch (Opcode) {
8458 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
8459 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
8460 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
8461 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
8462 default: return Register();
8463 }
8464}
8465
8466// FastEmit functions for ISD::ADD.
8467
8468Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_255_expr(MVT RetVT, Register Op0, uint64_t imm1) {
8469 if (RetVT.SimpleTy != MVT::i32)
8470 return Register();
8471 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
8472 return fastEmitInst_ri(MachineInstOpcode: ARM::tADDi8, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
8473 }
8474 return Register();
8475}
8476
8477Register fastEmit_ISD_ADD_ri_Predicate_imm0_255_expr(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8478 switch (VT.SimpleTy) {
8479 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_255_expr(RetVT, Op0, imm1);
8480 default: return Register();
8481 }
8482}
8483
8484// Top-level FastEmit function.
8485
8486Register fastEmit_ri_Predicate_imm0_255_expr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8487 switch (Opcode) {
8488 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm0_255_expr(VT, RetVT, Op0, imm1);
8489 default: return Register();
8490 }
8491}
8492
8493// FastEmit functions for ISD::ADD.
8494
8495Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8496 if (RetVT.SimpleTy != MVT::i32)
8497 return Register();
8498 if ((Subtarget->isThumb2())) {
8499 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ADDri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8500 }
8501 return Register();
8502}
8503
8504Register fastEmit_ISD_ADD_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8505 switch (VT.SimpleTy) {
8506 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8507 default: return Register();
8508 }
8509}
8510
8511// FastEmit functions for ISD::AND.
8512
8513Register fastEmit_ISD_AND_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8514 if (RetVT.SimpleTy != MVT::i32)
8515 return Register();
8516 if ((Subtarget->isThumb2())) {
8517 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ANDri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8518 }
8519 return Register();
8520}
8521
8522Register fastEmit_ISD_AND_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8523 switch (VT.SimpleTy) {
8524 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8525 default: return Register();
8526 }
8527}
8528
8529// FastEmit functions for ISD::OR.
8530
8531Register fastEmit_ISD_OR_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8532 if (RetVT.SimpleTy != MVT::i32)
8533 return Register();
8534 if ((Subtarget->isThumb2())) {
8535 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ORRri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8536 }
8537 return Register();
8538}
8539
8540Register fastEmit_ISD_OR_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8541 switch (VT.SimpleTy) {
8542 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8543 default: return Register();
8544 }
8545}
8546
8547// FastEmit functions for ISD::SUB.
8548
8549Register fastEmit_ISD_SUB_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8550 if (RetVT.SimpleTy != MVT::i32)
8551 return Register();
8552 if ((Subtarget->isThumb2())) {
8553 return fastEmitInst_ri(MachineInstOpcode: ARM::t2SUBri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8554 }
8555 return Register();
8556}
8557
8558Register fastEmit_ISD_SUB_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8559 switch (VT.SimpleTy) {
8560 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8561 default: return Register();
8562 }
8563}
8564
8565// FastEmit functions for ISD::XOR.
8566
8567Register fastEmit_ISD_XOR_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8568 if (RetVT.SimpleTy != MVT::i32)
8569 return Register();
8570 if ((Subtarget->isThumb2())) {
8571 return fastEmitInst_ri(MachineInstOpcode: ARM::t2EORri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8572 }
8573 return Register();
8574}
8575
8576Register fastEmit_ISD_XOR_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8577 switch (VT.SimpleTy) {
8578 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8579 default: return Register();
8580 }
8581}
8582
8583// Top-level FastEmit function.
8584
8585Register fastEmit_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8586 switch (Opcode) {
8587 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8588 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8589 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8590 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8591 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8592 default: return Register();
8593 }
8594}
8595
8596// FastEmit functions for ISD::ADD.
8597
8598Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_4095(MVT RetVT, Register Op0, uint64_t imm1) {
8599 if (RetVT.SimpleTy != MVT::i32)
8600 return Register();
8601 if ((Subtarget->isThumb2())) {
8602 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ADDri12, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8603 }
8604 return Register();
8605}
8606
8607Register fastEmit_ISD_ADD_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8608 switch (VT.SimpleTy) {
8609 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_4095(RetVT, Op0, imm1);
8610 default: return Register();
8611 }
8612}
8613
8614// FastEmit functions for ISD::SUB.
8615
8616Register fastEmit_ISD_SUB_MVT_i32_ri_Predicate_imm0_4095(MVT RetVT, Register Op0, uint64_t imm1) {
8617 if (RetVT.SimpleTy != MVT::i32)
8618 return Register();
8619 if ((Subtarget->isThumb2())) {
8620 return fastEmitInst_ri(MachineInstOpcode: ARM::t2SUBri12, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8621 }
8622 return Register();
8623}
8624
8625Register fastEmit_ISD_SUB_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8626 switch (VT.SimpleTy) {
8627 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_imm0_4095(RetVT, Op0, imm1);
8628 default: return Register();
8629 }
8630}
8631
8632// Top-level FastEmit function.
8633
8634Register fastEmit_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8635 switch (Opcode) {
8636 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm0_4095(VT, RetVT, Op0, imm1);
8637 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_imm0_4095(VT, RetVT, Op0, imm1);
8638 default: return Register();
8639 }
8640}
8641
8642// FastEmit functions for ISD::ROTR.
8643
8644Register fastEmit_ISD_ROTR_MVT_i32_ri_Predicate_imm1_31(MVT RetVT, Register Op0, uint64_t imm1) {
8645 if (RetVT.SimpleTy != MVT::i32)
8646 return Register();
8647 if ((Subtarget->isThumb2())) {
8648 return fastEmitInst_ri(MachineInstOpcode: ARM::t2RORri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8649 }
8650 return Register();
8651}
8652
8653Register fastEmit_ISD_ROTR_ri_Predicate_imm1_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8654 switch (VT.SimpleTy) {
8655 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_ri_Predicate_imm1_31(RetVT, Op0, imm1);
8656 default: return Register();
8657 }
8658}
8659
8660// FastEmit functions for ISD::SHL.
8661
8662Register fastEmit_ISD_SHL_MVT_i32_ri_Predicate_imm1_31(MVT RetVT, Register Op0, uint64_t imm1) {
8663 if (RetVT.SimpleTy != MVT::i32)
8664 return Register();
8665 if ((Subtarget->isThumb2())) {
8666 return fastEmitInst_ri(MachineInstOpcode: ARM::t2LSLri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8667 }
8668 return Register();
8669}
8670
8671Register fastEmit_ISD_SHL_ri_Predicate_imm1_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8672 switch (VT.SimpleTy) {
8673 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri_Predicate_imm1_31(RetVT, Op0, imm1);
8674 default: return Register();
8675 }
8676}
8677
8678// Top-level FastEmit function.
8679
8680Register fastEmit_ri_Predicate_imm1_31(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8681 switch (Opcode) {
8682 case ISD::ROTR: return fastEmit_ISD_ROTR_ri_Predicate_imm1_31(VT, RetVT, Op0, imm1);
8683 case ISD::SHL: return fastEmit_ISD_SHL_ri_Predicate_imm1_31(VT, RetVT, Op0, imm1);
8684 default: return Register();
8685 }
8686}
8687
8688// FastEmit functions for ARMISD::VQRSHRNsIMM.
8689
8690Register fastEmit_ARMISD_VQRSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8691 if (RetVT.SimpleTy != MVT::v8i8)
8692 return Register();
8693 if ((Subtarget->hasNEON())) {
8694 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8695 }
8696 return Register();
8697}
8698
8699Register fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8700 switch (VT.SimpleTy) {
8701 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8702 default: return Register();
8703 }
8704}
8705
8706// FastEmit functions for ARMISD::VQRSHRNsuIMM.
8707
8708Register fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8709 if (RetVT.SimpleTy != MVT::v8i8)
8710 return Register();
8711 if ((Subtarget->hasNEON())) {
8712 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRUNv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8713 }
8714 return Register();
8715}
8716
8717Register fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8718 switch (VT.SimpleTy) {
8719 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8720 default: return Register();
8721 }
8722}
8723
8724// FastEmit functions for ARMISD::VQRSHRNuIMM.
8725
8726Register fastEmit_ARMISD_VQRSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8727 if (RetVT.SimpleTy != MVT::v8i8)
8728 return Register();
8729 if ((Subtarget->hasNEON())) {
8730 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8731 }
8732 return Register();
8733}
8734
8735Register fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8736 switch (VT.SimpleTy) {
8737 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8738 default: return Register();
8739 }
8740}
8741
8742// FastEmit functions for ARMISD::VQSHRNsIMM.
8743
8744Register fastEmit_ARMISD_VQSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8745 if (RetVT.SimpleTy != MVT::v8i8)
8746 return Register();
8747 if ((Subtarget->hasNEON())) {
8748 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8749 }
8750 return Register();
8751}
8752
8753Register fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8754 switch (VT.SimpleTy) {
8755 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8756 default: return Register();
8757 }
8758}
8759
8760// FastEmit functions for ARMISD::VQSHRNsuIMM.
8761
8762Register fastEmit_ARMISD_VQSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8763 if (RetVT.SimpleTy != MVT::v8i8)
8764 return Register();
8765 if ((Subtarget->hasNEON())) {
8766 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRUNv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8767 }
8768 return Register();
8769}
8770
8771Register fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8772 switch (VT.SimpleTy) {
8773 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8774 default: return Register();
8775 }
8776}
8777
8778// FastEmit functions for ARMISD::VQSHRNuIMM.
8779
8780Register fastEmit_ARMISD_VQSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8781 if (RetVT.SimpleTy != MVT::v8i8)
8782 return Register();
8783 if ((Subtarget->hasNEON())) {
8784 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8785 }
8786 return Register();
8787}
8788
8789Register fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8790 switch (VT.SimpleTy) {
8791 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8792 default: return Register();
8793 }
8794}
8795
8796// FastEmit functions for ARMISD::VRSHRNIMM.
8797
8798Register fastEmit_ARMISD_VRSHRNIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8799 if (RetVT.SimpleTy != MVT::v8i8)
8800 return Register();
8801 if ((Subtarget->hasNEON())) {
8802 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRNv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8803 }
8804 return Register();
8805}
8806
8807Register fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8808 switch (VT.SimpleTy) {
8809 case MVT::v8i16: return fastEmit_ARMISD_VRSHRNIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8810 default: return Register();
8811 }
8812}
8813
8814// Top-level FastEmit function.
8815
8816Register fastEmit_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8817 switch (Opcode) {
8818 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8819 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8820 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8821 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8822 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8823 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8824 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8825 default: return Register();
8826 }
8827}
8828
8829// FastEmit functions for ARMISD::VQRSHRNsIMM.
8830
8831Register fastEmit_ARMISD_VQRSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
8832 if (RetVT.SimpleTy != MVT::v4i16)
8833 return Register();
8834 if ((Subtarget->hasNEON())) {
8835 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8836 }
8837 return Register();
8838}
8839
8840Register fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8841 switch (VT.SimpleTy) {
8842 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8843 default: return Register();
8844 }
8845}
8846
8847// FastEmit functions for ARMISD::VQRSHRNsuIMM.
8848
8849Register fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
8850 if (RetVT.SimpleTy != MVT::v4i16)
8851 return Register();
8852 if ((Subtarget->hasNEON())) {
8853 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRUNv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8854 }
8855 return Register();
8856}
8857
8858Register fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8859 switch (VT.SimpleTy) {
8860 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8861 default: return Register();
8862 }
8863}
8864
8865// FastEmit functions for ARMISD::VQRSHRNuIMM.
8866
8867Register fastEmit_ARMISD_VQRSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
8868 if (RetVT.SimpleTy != MVT::v4i16)
8869 return Register();
8870 if ((Subtarget->hasNEON())) {
8871 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8872 }
8873 return Register();
8874}
8875
8876Register fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8877 switch (VT.SimpleTy) {
8878 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8879 default: return Register();
8880 }
8881}
8882
8883// FastEmit functions for ARMISD::VQSHRNsIMM.
8884
8885Register fastEmit_ARMISD_VQSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
8886 if (RetVT.SimpleTy != MVT::v4i16)
8887 return Register();
8888 if ((Subtarget->hasNEON())) {
8889 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8890 }
8891 return Register();
8892}
8893
8894Register fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8895 switch (VT.SimpleTy) {
8896 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8897 default: return Register();
8898 }
8899}
8900
8901// FastEmit functions for ARMISD::VQSHRNsuIMM.
8902
8903Register fastEmit_ARMISD_VQSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
8904 if (RetVT.SimpleTy != MVT::v4i16)
8905 return Register();
8906 if ((Subtarget->hasNEON())) {
8907 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRUNv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8908 }
8909 return Register();
8910}
8911
8912Register fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8913 switch (VT.SimpleTy) {
8914 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8915 default: return Register();
8916 }
8917}
8918
8919// FastEmit functions for ARMISD::VQSHRNuIMM.
8920
8921Register fastEmit_ARMISD_VQSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
8922 if (RetVT.SimpleTy != MVT::v4i16)
8923 return Register();
8924 if ((Subtarget->hasNEON())) {
8925 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8926 }
8927 return Register();
8928}
8929
8930Register fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8931 switch (VT.SimpleTy) {
8932 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8933 default: return Register();
8934 }
8935}
8936
8937// FastEmit functions for ARMISD::VRSHRNIMM.
8938
8939Register fastEmit_ARMISD_VRSHRNIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
8940 if (RetVT.SimpleTy != MVT::v4i16)
8941 return Register();
8942 if ((Subtarget->hasNEON())) {
8943 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRNv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8944 }
8945 return Register();
8946}
8947
8948Register fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8949 switch (VT.SimpleTy) {
8950 case MVT::v4i32: return fastEmit_ARMISD_VRSHRNIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8951 default: return Register();
8952 }
8953}
8954
8955// Top-level FastEmit function.
8956
8957Register fastEmit_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8958 switch (Opcode) {
8959 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
8960 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
8961 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
8962 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
8963 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
8964 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
8965 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
8966 default: return Register();
8967 }
8968}
8969
8970// FastEmit functions for ARMISD::VQRSHRNsIMM.
8971
8972Register fastEmit_ARMISD_VQRSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
8973 if (RetVT.SimpleTy != MVT::v2i32)
8974 return Register();
8975 if ((Subtarget->hasNEON())) {
8976 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8977 }
8978 return Register();
8979}
8980
8981Register fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8982 switch (VT.SimpleTy) {
8983 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
8984 default: return Register();
8985 }
8986}
8987
8988// FastEmit functions for ARMISD::VQRSHRNsuIMM.
8989
8990Register fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
8991 if (RetVT.SimpleTy != MVT::v2i32)
8992 return Register();
8993 if ((Subtarget->hasNEON())) {
8994 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRUNv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8995 }
8996 return Register();
8997}
8998
8999Register fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9000 switch (VT.SimpleTy) {
9001 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9002 default: return Register();
9003 }
9004}
9005
9006// FastEmit functions for ARMISD::VQRSHRNuIMM.
9007
9008Register fastEmit_ARMISD_VQRSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9009 if (RetVT.SimpleTy != MVT::v2i32)
9010 return Register();
9011 if ((Subtarget->hasNEON())) {
9012 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9013 }
9014 return Register();
9015}
9016
9017Register fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9018 switch (VT.SimpleTy) {
9019 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9020 default: return Register();
9021 }
9022}
9023
9024// FastEmit functions for ARMISD::VQSHRNsIMM.
9025
9026Register fastEmit_ARMISD_VQSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9027 if (RetVT.SimpleTy != MVT::v2i32)
9028 return Register();
9029 if ((Subtarget->hasNEON())) {
9030 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9031 }
9032 return Register();
9033}
9034
9035Register fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9036 switch (VT.SimpleTy) {
9037 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9038 default: return Register();
9039 }
9040}
9041
9042// FastEmit functions for ARMISD::VQSHRNsuIMM.
9043
9044Register fastEmit_ARMISD_VQSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9045 if (RetVT.SimpleTy != MVT::v2i32)
9046 return Register();
9047 if ((Subtarget->hasNEON())) {
9048 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRUNv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9049 }
9050 return Register();
9051}
9052
9053Register fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9054 switch (VT.SimpleTy) {
9055 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9056 default: return Register();
9057 }
9058}
9059
9060// FastEmit functions for ARMISD::VQSHRNuIMM.
9061
9062Register fastEmit_ARMISD_VQSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9063 if (RetVT.SimpleTy != MVT::v2i32)
9064 return Register();
9065 if ((Subtarget->hasNEON())) {
9066 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9067 }
9068 return Register();
9069}
9070
9071Register fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9072 switch (VT.SimpleTy) {
9073 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9074 default: return Register();
9075 }
9076}
9077
9078// FastEmit functions for ARMISD::VRSHRNIMM.
9079
9080Register fastEmit_ARMISD_VRSHRNIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9081 if (RetVT.SimpleTy != MVT::v2i32)
9082 return Register();
9083 if ((Subtarget->hasNEON())) {
9084 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRNv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9085 }
9086 return Register();
9087}
9088
9089Register fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9090 switch (VT.SimpleTy) {
9091 case MVT::v2i64: return fastEmit_ARMISD_VRSHRNIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9092 default: return Register();
9093 }
9094}
9095
9096// Top-level FastEmit function.
9097
9098Register fastEmit_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9099 switch (Opcode) {
9100 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9101 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9102 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9103 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9104 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9105 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9106 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9107 default: return Register();
9108 }
9109}
9110
9111// FastEmit functions for ARMISD::VDUPLANE.
9112
9113Register fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri_Predicate_VectorIndex32(MVT RetVT, Register Op0, uint64_t imm1) {
9114 if (RetVT.SimpleTy != MVT::v16i8)
9115 return Register();
9116 if ((Subtarget->hasNEON())) {
9117 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN8q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
9118 }
9119 return Register();
9120}
9121
9122Register fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri_Predicate_VectorIndex32(MVT RetVT, Register Op0, uint64_t imm1) {
9123 if (RetVT.SimpleTy != MVT::v8i16)
9124 return Register();
9125 if ((Subtarget->hasNEON())) {
9126 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
9127 }
9128 return Register();
9129}
9130
9131Register fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri_Predicate_VectorIndex32(MVT RetVT, Register Op0, uint64_t imm1) {
9132 if (RetVT.SimpleTy != MVT::v4i32)
9133 return Register();
9134 if ((Subtarget->hasNEON())) {
9135 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
9136 }
9137 return Register();
9138}
9139
9140Register fastEmit_ARMISD_VDUPLANE_ri_Predicate_VectorIndex32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9141 switch (VT.SimpleTy) {
9142 case MVT::v8i8: return fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri_Predicate_VectorIndex32(RetVT, Op0, imm1);
9143 case MVT::v4i16: return fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri_Predicate_VectorIndex32(RetVT, Op0, imm1);
9144 case MVT::v2i32: return fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri_Predicate_VectorIndex32(RetVT, Op0, imm1);
9145 default: return Register();
9146 }
9147}
9148
9149// Top-level FastEmit function.
9150
9151Register fastEmit_ri_Predicate_VectorIndex32(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9152 switch (Opcode) {
9153 case ARMISD::VDUPLANE: return fastEmit_ARMISD_VDUPLANE_ri_Predicate_VectorIndex32(VT, RetVT, Op0, imm1);
9154 default: return Register();
9155 }
9156}
9157
9158// FastEmit functions for ARMISD::VSHLIMM.
9159
9160Register fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, Register Op0, uint64_t imm1) {
9161 if (RetVT.SimpleTy != MVT::v4i32)
9162 return Register();
9163 if ((Subtarget->hasMVEIntegerOps())) {
9164 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHL_immi32, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9165 }
9166 return Register();
9167}
9168
9169Register fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9170 switch (VT.SimpleTy) {
9171 case MVT::v4i32: return fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, imm1);
9172 default: return Register();
9173 }
9174}
9175
9176// FastEmit functions for ARMISD::VSHRsIMM.
9177
9178Register fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, Register Op0, uint64_t imm1) {
9179 if (RetVT.SimpleTy != MVT::v4i32)
9180 return Register();
9181 if ((Subtarget->hasMVEIntegerOps())) {
9182 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_imms32, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9183 }
9184 return Register();
9185}
9186
9187Register fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9188 switch (VT.SimpleTy) {
9189 case MVT::v4i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, imm1);
9190 default: return Register();
9191 }
9192}
9193
9194// FastEmit functions for ARMISD::VSHRuIMM.
9195
9196Register fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, Register Op0, uint64_t imm1) {
9197 if (RetVT.SimpleTy != MVT::v4i32)
9198 return Register();
9199 if ((Subtarget->hasMVEIntegerOps())) {
9200 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_immu32, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9201 }
9202 return Register();
9203}
9204
9205Register fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9206 switch (VT.SimpleTy) {
9207 case MVT::v4i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, imm1);
9208 default: return Register();
9209 }
9210}
9211
9212// Top-level FastEmit function.
9213
9214Register fastEmit_ri_Predicate_imm0_31(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9215 switch (Opcode) {
9216 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, imm1);
9217 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, imm1);
9218 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, imm1);
9219 default: return Register();
9220 }
9221}
9222
9223// FastEmit functions for ARMISD::VSHLIMM.
9224
9225Register fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, Register Op0, uint64_t imm1) {
9226 if (RetVT.SimpleTy != MVT::v8i16)
9227 return Register();
9228 if ((Subtarget->hasMVEIntegerOps())) {
9229 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHL_immi16, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9230 }
9231 return Register();
9232}
9233
9234Register fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9235 switch (VT.SimpleTy) {
9236 case MVT::v8i16: return fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, imm1);
9237 default: return Register();
9238 }
9239}
9240
9241// FastEmit functions for ARMISD::VSHRsIMM.
9242
9243Register fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, Register Op0, uint64_t imm1) {
9244 if (RetVT.SimpleTy != MVT::v8i16)
9245 return Register();
9246 if ((Subtarget->hasMVEIntegerOps())) {
9247 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_imms16, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9248 }
9249 return Register();
9250}
9251
9252Register fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9253 switch (VT.SimpleTy) {
9254 case MVT::v8i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, imm1);
9255 default: return Register();
9256 }
9257}
9258
9259// FastEmit functions for ARMISD::VSHRuIMM.
9260
9261Register fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, Register Op0, uint64_t imm1) {
9262 if (RetVT.SimpleTy != MVT::v8i16)
9263 return Register();
9264 if ((Subtarget->hasMVEIntegerOps())) {
9265 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_immu16, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9266 }
9267 return Register();
9268}
9269
9270Register fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9271 switch (VT.SimpleTy) {
9272 case MVT::v8i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, imm1);
9273 default: return Register();
9274 }
9275}
9276
9277// Top-level FastEmit function.
9278
9279Register fastEmit_ri_Predicate_imm0_15(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9280 switch (Opcode) {
9281 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, imm1);
9282 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, imm1);
9283 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, imm1);
9284 default: return Register();
9285 }
9286}
9287
9288// FastEmit functions for ISD::Constant.
9289
9290Register fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
9291 if (RetVT.SimpleTy != MVT::i32)
9292 return Register();
9293 if ((Subtarget->isThumb()) && (Subtarget->useMovt())) {
9294 return fastEmitInst_i(MachineInstOpcode: ARM::t2MOVi32imm, RC: &ARM::rGPRRegClass, Imm: imm0);
9295 }
9296 if ((!Subtarget->useMovt()) && (Subtarget->genExecuteOnly()) && (Subtarget->isThumb1Only())) {
9297 return fastEmitInst_i(MachineInstOpcode: ARM::tMOVi32imm, RC: &ARM::rGPRRegClass, Imm: imm0);
9298 }
9299 return Register();
9300}
9301
9302Register fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
9303 switch (VT.SimpleTy) {
9304 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
9305 default: return Register();
9306 }
9307}
9308
9309// Top-level FastEmit function.
9310
9311Register fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
9312 switch (Opcode) {
9313 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
9314 default: return Register();
9315 }
9316}
9317
9318