1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the ARM target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_mod_imm(int64_t Imm) {
12
13 return ARM_AM::getSOImmVal(Arg: Imm) != -1;
14
15}
16static bool Predicate_imm0_7(int64_t Imm) {
17
18 return Imm >= 0 && Imm < 8;
19
20}
21static bool Predicate_imm0_255_expr(int64_t Imm) {
22 return Imm >= 0 && Imm < 256;
23}
24static bool Predicate_imm_sr(int64_t Imm) {
25
26 return Imm > 0 && Imm <= 32;
27
28}
29static bool Predicate_imm0_255(int64_t Imm) {
30 return Imm >= 0 && Imm < 256;
31}
32static bool Predicate_t2_so_imm(int64_t Imm) {
33
34 return ARM_AM::getT2SOImmVal(Arg: Imm) != -1;
35
36}
37static bool Predicate_imm0_4095(int64_t Imm) {
38
39 return Imm >= 0 && Imm < 4096;
40
41}
42static bool Predicate_imm1_31(int64_t Imm) {
43 return Imm > 0 && Imm < 32;
44}
45static bool Predicate_shr_imm8(int64_t Imm) {
46 return Imm > 0 && Imm <= 8;
47}
48static bool Predicate_shr_imm16(int64_t Imm) {
49 return Imm > 0 && Imm <= 16;
50}
51static bool Predicate_shr_imm32(int64_t Imm) {
52 return Imm > 0 && Imm <= 32;
53}
54static bool Predicate_VectorIndex32(int64_t Imm) {
55
56 return ((uint64_t)Imm) < 2;
57
58}
59static bool Predicate_imm0_31(int64_t Imm) {
60
61 return Imm >= 0 && Imm < 32;
62
63}
64static bool Predicate_mod_imm_not(int64_t Imm) {
65
66 return ARM_AM::getSOImmVal(Arg: ~(uint32_t)Imm) != -1;
67
68}
69static bool Predicate_imm0_65535(int64_t Imm) {
70
71 return Imm >= 0 && Imm < 65536;
72
73}
74static bool Predicate_t2_so_imm_neg(int64_t Imm) {
75
76 return Imm && ARM_AM::getT2SOImmVal(Arg: -(uint32_t)Imm) != -1;
77
78}
79static bool Predicate_imm0_15(int64_t Imm) {
80
81 return Imm >= 0 && Imm < 16;
82
83}
84
85
86// FastEmit functions for ISD::GET_FPENV.
87
88Register fastEmit_ISD_GET_FPENV_MVT_i32_(MVT RetVT) {
89 if (RetVT.SimpleTy != MVT::i32)
90 return Register();
91 return fastEmitInst_(MachineInstOpcode: ARM::VMRS, RC: &ARM::GPRnopcRegClass);
92}
93
94Register fastEmit_ISD_GET_FPENV_(MVT VT, MVT RetVT) {
95 switch (VT.SimpleTy) {
96 case MVT::i32: return fastEmit_ISD_GET_FPENV_MVT_i32_(RetVT);
97 default: return Register();
98 }
99}
100
101// FastEmit functions for ISD::GET_FPMODE.
102
103Register fastEmit_ISD_GET_FPMODE_MVT_i32_(MVT RetVT) {
104 if (RetVT.SimpleTy != MVT::i32)
105 return Register();
106 return fastEmitInst_(MachineInstOpcode: ARM::VMRS, RC: &ARM::GPRnopcRegClass);
107}
108
109Register fastEmit_ISD_GET_FPMODE_(MVT VT, MVT RetVT) {
110 switch (VT.SimpleTy) {
111 case MVT::i32: return fastEmit_ISD_GET_FPMODE_MVT_i32_(RetVT);
112 default: return Register();
113 }
114}
115
116// Top-level FastEmit function.
117
118Register fastEmit_(MVT VT, MVT RetVT, unsigned Opcode) override {
119 switch (Opcode) {
120 case ISD::GET_FPENV: return fastEmit_ISD_GET_FPENV_(VT, RetVT);
121 case ISD::GET_FPMODE: return fastEmit_ISD_GET_FPMODE_(VT, RetVT);
122 default: return Register();
123 }
124}
125
126// FastEmit functions for ARMISD::CALL.
127
128Register fastEmit_ARMISD_CALL_MVT_i32_r(MVT RetVT, Register Op0) {
129 if (RetVT.SimpleTy != MVT::isVoid)
130 return Register();
131 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
132 return fastEmitInst_r(MachineInstOpcode: ARM::BLX_noip, RC: &ARM::GPRnoipRegClass, Op0);
133 }
134 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( !MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
135 return fastEmitInst_r(MachineInstOpcode: ARM::BLX, RC: &ARM::GPRRegClass, Op0);
136 }
137 return Register();
138}
139
140Register fastEmit_ARMISD_CALL_r(MVT VT, MVT RetVT, Register Op0) {
141 switch (VT.SimpleTy) {
142 case MVT::i32: return fastEmit_ARMISD_CALL_MVT_i32_r(RetVT, Op0);
143 default: return Register();
144 }
145}
146
147// FastEmit functions for ARMISD::CALL_NOLINK.
148
149Register fastEmit_ARMISD_CALL_NOLINK_MVT_i32_r(MVT RetVT, Register Op0) {
150 if (RetVT.SimpleTy != MVT::isVoid)
151 return Register();
152 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
153 return fastEmitInst_r(MachineInstOpcode: ARM::tBX_CALL, RC: &ARM::tGPRRegClass, Op0);
154 }
155 if ((!Subtarget->isThumb()) && (!Subtarget->hasV4TOps())) {
156 return fastEmitInst_r(MachineInstOpcode: ARM::BMOVPCRX_CALL, RC: &ARM::tGPRRegClass, Op0);
157 }
158 if ((Subtarget->hasV4TOps()) && (!Subtarget->isThumb())) {
159 return fastEmitInst_r(MachineInstOpcode: ARM::BX_CALL, RC: &ARM::tGPRRegClass, Op0);
160 }
161 return Register();
162}
163
164Register fastEmit_ARMISD_CALL_NOLINK_r(MVT VT, MVT RetVT, Register Op0) {
165 switch (VT.SimpleTy) {
166 case MVT::i32: return fastEmit_ARMISD_CALL_NOLINK_MVT_i32_r(RetVT, Op0);
167 default: return Register();
168 }
169}
170
171// FastEmit functions for ARMISD::CALL_PRED.
172
173Register fastEmit_ARMISD_CALL_PRED_MVT_i32_r(MVT RetVT, Register Op0) {
174 if (RetVT.SimpleTy != MVT::isVoid)
175 return Register();
176 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
177 return fastEmitInst_r(MachineInstOpcode: ARM::BLX_pred_noip, RC: &ARM::GPRnoipRegClass, Op0);
178 }
179 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( !MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
180 return fastEmitInst_r(MachineInstOpcode: ARM::BLX_pred, RC: &ARM::GPRRegClass, Op0);
181 }
182 return Register();
183}
184
185Register fastEmit_ARMISD_CALL_PRED_r(MVT VT, MVT RetVT, Register Op0) {
186 switch (VT.SimpleTy) {
187 case MVT::i32: return fastEmit_ARMISD_CALL_PRED_MVT_i32_r(RetVT, Op0);
188 default: return Register();
189 }
190}
191
192// FastEmit functions for ARMISD::CMPFPEw0.
193
194Register fastEmit_ARMISD_CMPFPEw0_MVT_f16_r(MVT RetVT, Register Op0) {
195 if (RetVT.SimpleTy != MVT::i32)
196 return Register();
197 if ((Subtarget->hasFullFP16())) {
198 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPEZH, RC: &ARM::HPRRegClass, Op0);
199 }
200 return Register();
201}
202
203Register fastEmit_ARMISD_CMPFPEw0_MVT_f32_r(MVT RetVT, Register Op0) {
204 if (RetVT.SimpleTy != MVT::i32)
205 return Register();
206 if ((Subtarget->hasVFP2Base())) {
207 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPEZS, RC: &ARM::SPRRegClass, Op0);
208 }
209 return Register();
210}
211
212Register fastEmit_ARMISD_CMPFPEw0_MVT_f64_r(MVT RetVT, Register Op0) {
213 if (RetVT.SimpleTy != MVT::i32)
214 return Register();
215 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
216 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPEZD, RC: &ARM::DPRRegClass, Op0);
217 }
218 return Register();
219}
220
221Register fastEmit_ARMISD_CMPFPEw0_r(MVT VT, MVT RetVT, Register Op0) {
222 switch (VT.SimpleTy) {
223 case MVT::f16: return fastEmit_ARMISD_CMPFPEw0_MVT_f16_r(RetVT, Op0);
224 case MVT::f32: return fastEmit_ARMISD_CMPFPEw0_MVT_f32_r(RetVT, Op0);
225 case MVT::f64: return fastEmit_ARMISD_CMPFPEw0_MVT_f64_r(RetVT, Op0);
226 default: return Register();
227 }
228}
229
230// FastEmit functions for ARMISD::CMPFPw0.
231
232Register fastEmit_ARMISD_CMPFPw0_MVT_f16_r(MVT RetVT, Register Op0) {
233 if (RetVT.SimpleTy != MVT::i32)
234 return Register();
235 if ((Subtarget->hasFullFP16())) {
236 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPZH, RC: &ARM::HPRRegClass, Op0);
237 }
238 return Register();
239}
240
241Register fastEmit_ARMISD_CMPFPw0_MVT_f32_r(MVT RetVT, Register Op0) {
242 if (RetVT.SimpleTy != MVT::i32)
243 return Register();
244 if ((Subtarget->hasVFP2Base())) {
245 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPZS, RC: &ARM::SPRRegClass, Op0);
246 }
247 return Register();
248}
249
250Register fastEmit_ARMISD_CMPFPw0_MVT_f64_r(MVT RetVT, Register Op0) {
251 if (RetVT.SimpleTy != MVT::i32)
252 return Register();
253 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
254 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPZD, RC: &ARM::DPRRegClass, Op0);
255 }
256 return Register();
257}
258
259Register fastEmit_ARMISD_CMPFPw0_r(MVT VT, MVT RetVT, Register Op0) {
260 switch (VT.SimpleTy) {
261 case MVT::f16: return fastEmit_ARMISD_CMPFPw0_MVT_f16_r(RetVT, Op0);
262 case MVT::f32: return fastEmit_ARMISD_CMPFPw0_MVT_f32_r(RetVT, Op0);
263 case MVT::f64: return fastEmit_ARMISD_CMPFPw0_MVT_f64_r(RetVT, Op0);
264 default: return Register();
265 }
266}
267
268// FastEmit functions for ARMISD::VADDVs.
269
270Register fastEmit_ARMISD_VADDVs_MVT_v16i8_r(MVT RetVT, Register Op0) {
271 if (RetVT.SimpleTy != MVT::i32)
272 return Register();
273 if ((Subtarget->hasMVEIntegerOps())) {
274 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVs8no_acc, RC: &ARM::tGPREvenRegClass, Op0);
275 }
276 return Register();
277}
278
279Register fastEmit_ARMISD_VADDVs_MVT_v8i16_r(MVT RetVT, Register Op0) {
280 if (RetVT.SimpleTy != MVT::i32)
281 return Register();
282 if ((Subtarget->hasMVEIntegerOps())) {
283 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVs16no_acc, RC: &ARM::tGPREvenRegClass, Op0);
284 }
285 return Register();
286}
287
288Register fastEmit_ARMISD_VADDVs_MVT_v4i32_r(MVT RetVT, Register Op0) {
289 if (RetVT.SimpleTy != MVT::i32)
290 return Register();
291 if ((Subtarget->hasMVEIntegerOps())) {
292 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVs32no_acc, RC: &ARM::tGPREvenRegClass, Op0);
293 }
294 return Register();
295}
296
297Register fastEmit_ARMISD_VADDVs_r(MVT VT, MVT RetVT, Register Op0) {
298 switch (VT.SimpleTy) {
299 case MVT::v16i8: return fastEmit_ARMISD_VADDVs_MVT_v16i8_r(RetVT, Op0);
300 case MVT::v8i16: return fastEmit_ARMISD_VADDVs_MVT_v8i16_r(RetVT, Op0);
301 case MVT::v4i32: return fastEmit_ARMISD_VADDVs_MVT_v4i32_r(RetVT, Op0);
302 default: return Register();
303 }
304}
305
306// FastEmit functions for ARMISD::VADDVu.
307
308Register fastEmit_ARMISD_VADDVu_MVT_v16i8_r(MVT RetVT, Register Op0) {
309 if (RetVT.SimpleTy != MVT::i32)
310 return Register();
311 if ((Subtarget->hasMVEIntegerOps())) {
312 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu8no_acc, RC: &ARM::tGPREvenRegClass, Op0);
313 }
314 return Register();
315}
316
317Register fastEmit_ARMISD_VADDVu_MVT_v8i16_r(MVT RetVT, Register Op0) {
318 if (RetVT.SimpleTy != MVT::i32)
319 return Register();
320 if ((Subtarget->hasMVEIntegerOps())) {
321 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu16no_acc, RC: &ARM::tGPREvenRegClass, Op0);
322 }
323 return Register();
324}
325
326Register fastEmit_ARMISD_VADDVu_MVT_v4i32_r(MVT RetVT, Register Op0) {
327 if (RetVT.SimpleTy != MVT::i32)
328 return Register();
329 if ((Subtarget->hasMVEIntegerOps())) {
330 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu32no_acc, RC: &ARM::tGPREvenRegClass, Op0);
331 }
332 return Register();
333}
334
335Register fastEmit_ARMISD_VADDVu_r(MVT VT, MVT RetVT, Register Op0) {
336 switch (VT.SimpleTy) {
337 case MVT::v16i8: return fastEmit_ARMISD_VADDVu_MVT_v16i8_r(RetVT, Op0);
338 case MVT::v8i16: return fastEmit_ARMISD_VADDVu_MVT_v8i16_r(RetVT, Op0);
339 case MVT::v4i32: return fastEmit_ARMISD_VADDVu_MVT_v4i32_r(RetVT, Op0);
340 default: return Register();
341 }
342}
343
344// FastEmit functions for ARMISD::VDUP.
345
346Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i8_r(Register Op0) {
347 if ((Subtarget->hasNEON())) {
348 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP8d, RC: &ARM::DPRRegClass, Op0);
349 }
350 return Register();
351}
352
353Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v16i8_r(Register Op0) {
354 if ((Subtarget->hasMVEIntegerOps())) {
355 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP8, RC: &ARM::MQPRRegClass, Op0);
356 }
357 if ((Subtarget->hasNEON())) {
358 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP8q, RC: &ARM::QPRRegClass, Op0);
359 }
360 return Register();
361}
362
363Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i16_r(Register Op0) {
364 if ((Subtarget->hasNEON())) {
365 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP16d, RC: &ARM::DPRRegClass, Op0);
366 }
367 return Register();
368}
369
370Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i16_r(Register Op0) {
371 if ((Subtarget->hasMVEIntegerOps())) {
372 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP16, RC: &ARM::MQPRRegClass, Op0);
373 }
374 if ((Subtarget->hasNEON())) {
375 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP16q, RC: &ARM::QPRRegClass, Op0);
376 }
377 return Register();
378}
379
380Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v2i32_r(Register Op0) {
381 if ((!Subtarget->hasSlowVDUP32()) && (Subtarget->hasNEON())) {
382 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP32d, RC: &ARM::DPRRegClass, Op0);
383 }
384 return Register();
385}
386
387Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i32_r(Register Op0) {
388 if ((Subtarget->hasMVEIntegerOps())) {
389 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP32, RC: &ARM::MQPRRegClass, Op0);
390 }
391 if ((Subtarget->hasNEON())) {
392 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP32q, RC: &ARM::QPRRegClass, Op0);
393 }
394 return Register();
395}
396
397Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8f16_r(Register Op0) {
398 if ((Subtarget->hasMVEIntegerOps())) {
399 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP16, RC: &ARM::MQPRRegClass, Op0);
400 }
401 return Register();
402}
403
404Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4f32_r(Register Op0) {
405 if ((Subtarget->hasMVEIntegerOps())) {
406 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP32, RC: &ARM::MQPRRegClass, Op0);
407 }
408 return Register();
409}
410
411Register fastEmit_ARMISD_VDUP_MVT_i32_r(MVT RetVT, Register Op0) {
412switch (RetVT.SimpleTy) {
413 case MVT::v8i8: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i8_r(Op0);
414 case MVT::v16i8: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v16i8_r(Op0);
415 case MVT::v4i16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i16_r(Op0);
416 case MVT::v8i16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i16_r(Op0);
417 case MVT::v2i32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v2i32_r(Op0);
418 case MVT::v4i32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i32_r(Op0);
419 case MVT::v8f16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8f16_r(Op0);
420 case MVT::v4f32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4f32_r(Op0);
421 default: return Register();
422}
423}
424
425Register fastEmit_ARMISD_VDUP_r(MVT VT, MVT RetVT, Register Op0) {
426 switch (VT.SimpleTy) {
427 case MVT::i32: return fastEmit_ARMISD_VDUP_MVT_i32_r(RetVT, Op0);
428 default: return Register();
429 }
430}
431
432// FastEmit functions for ARMISD::VMOVSR.
433
434Register fastEmit_ARMISD_VMOVSR_MVT_i32_r(MVT RetVT, Register Op0) {
435 if (RetVT.SimpleTy != MVT::f32)
436 return Register();
437 if ((Subtarget->hasFPRegs()) && (Subtarget->preferVMOVSR() ||!Subtarget->useNEONForSinglePrecisionFP())) {
438 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVSR, RC: &ARM::SPRRegClass, Op0);
439 }
440 return Register();
441}
442
443Register fastEmit_ARMISD_VMOVSR_r(MVT VT, MVT RetVT, Register Op0) {
444 switch (VT.SimpleTy) {
445 case MVT::i32: return fastEmit_ARMISD_VMOVSR_MVT_i32_r(RetVT, Op0);
446 default: return Register();
447 }
448}
449
450// FastEmit functions for ARMISD::VMOVhr.
451
452Register fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_bf16_r(Register Op0) {
453 if ((Subtarget->hasFPRegs16())) {
454 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVHR, RC: &ARM::HPRRegClass, Op0);
455 }
456 return Register();
457}
458
459Register fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_f16_r(Register Op0) {
460 if ((Subtarget->hasFPRegs16())) {
461 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVHR, RC: &ARM::HPRRegClass, Op0);
462 }
463 return Register();
464}
465
466Register fastEmit_ARMISD_VMOVhr_MVT_i32_r(MVT RetVT, Register Op0) {
467switch (RetVT.SimpleTy) {
468 case MVT::bf16: return fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_bf16_r(Op0);
469 case MVT::f16: return fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_f16_r(Op0);
470 default: return Register();
471}
472}
473
474Register fastEmit_ARMISD_VMOVhr_r(MVT VT, MVT RetVT, Register Op0) {
475 switch (VT.SimpleTy) {
476 case MVT::i32: return fastEmit_ARMISD_VMOVhr_MVT_i32_r(RetVT, Op0);
477 default: return Register();
478 }
479}
480
481// FastEmit functions for ARMISD::VMOVrh.
482
483Register fastEmit_ARMISD_VMOVrh_MVT_bf16_r(MVT RetVT, Register Op0) {
484 if (RetVT.SimpleTy != MVT::i32)
485 return Register();
486 if ((Subtarget->hasFPRegs16())) {
487 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVRH, RC: &ARM::rGPRRegClass, Op0);
488 }
489 return Register();
490}
491
492Register fastEmit_ARMISD_VMOVrh_MVT_f16_r(MVT RetVT, Register Op0) {
493 if (RetVT.SimpleTy != MVT::i32)
494 return Register();
495 if ((Subtarget->hasFPRegs16())) {
496 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVRH, RC: &ARM::rGPRRegClass, Op0);
497 }
498 return Register();
499}
500
501Register fastEmit_ARMISD_VMOVrh_r(MVT VT, MVT RetVT, Register Op0) {
502 switch (VT.SimpleTy) {
503 case MVT::bf16: return fastEmit_ARMISD_VMOVrh_MVT_bf16_r(RetVT, Op0);
504 case MVT::f16: return fastEmit_ARMISD_VMOVrh_MVT_f16_r(RetVT, Op0);
505 default: return Register();
506 }
507}
508
509// FastEmit functions for ARMISD::VREV16.
510
511Register fastEmit_ARMISD_VREV16_MVT_v8i8_r(MVT RetVT, Register Op0) {
512 if (RetVT.SimpleTy != MVT::v8i8)
513 return Register();
514 if ((Subtarget->hasNEON())) {
515 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
516 }
517 return Register();
518}
519
520Register fastEmit_ARMISD_VREV16_MVT_v16i8_r(MVT RetVT, Register Op0) {
521 if (RetVT.SimpleTy != MVT::v16i8)
522 return Register();
523 if ((Subtarget->hasMVEIntegerOps())) {
524 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
525 }
526 if ((Subtarget->hasNEON())) {
527 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
528 }
529 return Register();
530}
531
532Register fastEmit_ARMISD_VREV16_r(MVT VT, MVT RetVT, Register Op0) {
533 switch (VT.SimpleTy) {
534 case MVT::v8i8: return fastEmit_ARMISD_VREV16_MVT_v8i8_r(RetVT, Op0);
535 case MVT::v16i8: return fastEmit_ARMISD_VREV16_MVT_v16i8_r(RetVT, Op0);
536 default: return Register();
537 }
538}
539
540// FastEmit functions for ARMISD::VREV32.
541
542Register fastEmit_ARMISD_VREV32_MVT_v8i8_r(MVT RetVT, Register Op0) {
543 if (RetVT.SimpleTy != MVT::v8i8)
544 return Register();
545 if ((Subtarget->hasNEON())) {
546 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
547 }
548 return Register();
549}
550
551Register fastEmit_ARMISD_VREV32_MVT_v16i8_r(MVT RetVT, Register Op0) {
552 if (RetVT.SimpleTy != MVT::v16i8)
553 return Register();
554 if ((Subtarget->hasMVEIntegerOps())) {
555 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
556 }
557 if ((Subtarget->hasNEON())) {
558 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
559 }
560 return Register();
561}
562
563Register fastEmit_ARMISD_VREV32_MVT_v4i16_r(MVT RetVT, Register Op0) {
564 if (RetVT.SimpleTy != MVT::v4i16)
565 return Register();
566 if ((Subtarget->hasNEON())) {
567 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
568 }
569 return Register();
570}
571
572Register fastEmit_ARMISD_VREV32_MVT_v8i16_r(MVT RetVT, Register Op0) {
573 if (RetVT.SimpleTy != MVT::v8i16)
574 return Register();
575 if ((Subtarget->hasMVEIntegerOps())) {
576 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
577 }
578 if ((Subtarget->hasNEON())) {
579 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
580 }
581 return Register();
582}
583
584Register fastEmit_ARMISD_VREV32_MVT_v4f16_r(MVT RetVT, Register Op0) {
585 if (RetVT.SimpleTy != MVT::v4f16)
586 return Register();
587 if ((Subtarget->hasNEON())) {
588 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
589 }
590 return Register();
591}
592
593Register fastEmit_ARMISD_VREV32_MVT_v8f16_r(MVT RetVT, Register Op0) {
594 if (RetVT.SimpleTy != MVT::v8f16)
595 return Register();
596 if ((Subtarget->hasMVEIntegerOps())) {
597 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
598 }
599 if ((Subtarget->hasNEON())) {
600 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
601 }
602 return Register();
603}
604
605Register fastEmit_ARMISD_VREV32_MVT_v4bf16_r(MVT RetVT, Register Op0) {
606 if (RetVT.SimpleTy != MVT::v4bf16)
607 return Register();
608 if ((Subtarget->hasNEON())) {
609 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
610 }
611 return Register();
612}
613
614Register fastEmit_ARMISD_VREV32_MVT_v8bf16_r(MVT RetVT, Register Op0) {
615 if (RetVT.SimpleTy != MVT::v8bf16)
616 return Register();
617 if ((Subtarget->hasNEON())) {
618 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
619 }
620 return Register();
621}
622
623Register fastEmit_ARMISD_VREV32_r(MVT VT, MVT RetVT, Register Op0) {
624 switch (VT.SimpleTy) {
625 case MVT::v8i8: return fastEmit_ARMISD_VREV32_MVT_v8i8_r(RetVT, Op0);
626 case MVT::v16i8: return fastEmit_ARMISD_VREV32_MVT_v16i8_r(RetVT, Op0);
627 case MVT::v4i16: return fastEmit_ARMISD_VREV32_MVT_v4i16_r(RetVT, Op0);
628 case MVT::v8i16: return fastEmit_ARMISD_VREV32_MVT_v8i16_r(RetVT, Op0);
629 case MVT::v4f16: return fastEmit_ARMISD_VREV32_MVT_v4f16_r(RetVT, Op0);
630 case MVT::v8f16: return fastEmit_ARMISD_VREV32_MVT_v8f16_r(RetVT, Op0);
631 case MVT::v4bf16: return fastEmit_ARMISD_VREV32_MVT_v4bf16_r(RetVT, Op0);
632 case MVT::v8bf16: return fastEmit_ARMISD_VREV32_MVT_v8bf16_r(RetVT, Op0);
633 default: return Register();
634 }
635}
636
637// FastEmit functions for ARMISD::VREV64.
638
639Register fastEmit_ARMISD_VREV64_MVT_v8i8_r(MVT RetVT, Register Op0) {
640 if (RetVT.SimpleTy != MVT::v8i8)
641 return Register();
642 if ((Subtarget->hasNEON())) {
643 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
644 }
645 return Register();
646}
647
648Register fastEmit_ARMISD_VREV64_MVT_v16i8_r(MVT RetVT, Register Op0) {
649 if (RetVT.SimpleTy != MVT::v16i8)
650 return Register();
651 if ((Subtarget->hasMVEIntegerOps())) {
652 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
653 }
654 if ((Subtarget->hasNEON())) {
655 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
656 }
657 return Register();
658}
659
660Register fastEmit_ARMISD_VREV64_MVT_v4i16_r(MVT RetVT, Register Op0) {
661 if (RetVT.SimpleTy != MVT::v4i16)
662 return Register();
663 if ((Subtarget->hasNEON())) {
664 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
665 }
666 return Register();
667}
668
669Register fastEmit_ARMISD_VREV64_MVT_v8i16_r(MVT RetVT, Register Op0) {
670 if (RetVT.SimpleTy != MVT::v8i16)
671 return Register();
672 if ((Subtarget->hasMVEIntegerOps())) {
673 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
674 }
675 if ((Subtarget->hasNEON())) {
676 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
677 }
678 return Register();
679}
680
681Register fastEmit_ARMISD_VREV64_MVT_v2i32_r(MVT RetVT, Register Op0) {
682 if (RetVT.SimpleTy != MVT::v2i32)
683 return Register();
684 if ((Subtarget->hasNEON())) {
685 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
686 }
687 return Register();
688}
689
690Register fastEmit_ARMISD_VREV64_MVT_v4i32_r(MVT RetVT, Register Op0) {
691 if (RetVT.SimpleTy != MVT::v4i32)
692 return Register();
693 if ((Subtarget->hasMVEIntegerOps())) {
694 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
695 }
696 if ((Subtarget->hasNEON())) {
697 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
698 }
699 return Register();
700}
701
702Register fastEmit_ARMISD_VREV64_MVT_v4f16_r(MVT RetVT, Register Op0) {
703 if (RetVT.SimpleTy != MVT::v4f16)
704 return Register();
705 if ((Subtarget->hasNEON())) {
706 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
707 }
708 return Register();
709}
710
711Register fastEmit_ARMISD_VREV64_MVT_v8f16_r(MVT RetVT, Register Op0) {
712 if (RetVT.SimpleTy != MVT::v8f16)
713 return Register();
714 if ((Subtarget->hasMVEIntegerOps())) {
715 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
716 }
717 if ((Subtarget->hasNEON())) {
718 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
719 }
720 return Register();
721}
722
723Register fastEmit_ARMISD_VREV64_MVT_v4bf16_r(MVT RetVT, Register Op0) {
724 if (RetVT.SimpleTy != MVT::v4bf16)
725 return Register();
726 if ((Subtarget->hasNEON())) {
727 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
728 }
729 return Register();
730}
731
732Register fastEmit_ARMISD_VREV64_MVT_v8bf16_r(MVT RetVT, Register Op0) {
733 if (RetVT.SimpleTy != MVT::v8bf16)
734 return Register();
735 if ((Subtarget->hasNEON())) {
736 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
737 }
738 return Register();
739}
740
741Register fastEmit_ARMISD_VREV64_MVT_v2f32_r(MVT RetVT, Register Op0) {
742 if (RetVT.SimpleTy != MVT::v2f32)
743 return Register();
744 if ((Subtarget->hasNEON())) {
745 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
746 }
747 return Register();
748}
749
750Register fastEmit_ARMISD_VREV64_MVT_v4f32_r(MVT RetVT, Register Op0) {
751 if (RetVT.SimpleTy != MVT::v4f32)
752 return Register();
753 if ((Subtarget->hasMVEIntegerOps())) {
754 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
755 }
756 if ((Subtarget->hasNEON())) {
757 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
758 }
759 return Register();
760}
761
762Register fastEmit_ARMISD_VREV64_r(MVT VT, MVT RetVT, Register Op0) {
763 switch (VT.SimpleTy) {
764 case MVT::v8i8: return fastEmit_ARMISD_VREV64_MVT_v8i8_r(RetVT, Op0);
765 case MVT::v16i8: return fastEmit_ARMISD_VREV64_MVT_v16i8_r(RetVT, Op0);
766 case MVT::v4i16: return fastEmit_ARMISD_VREV64_MVT_v4i16_r(RetVT, Op0);
767 case MVT::v8i16: return fastEmit_ARMISD_VREV64_MVT_v8i16_r(RetVT, Op0);
768 case MVT::v2i32: return fastEmit_ARMISD_VREV64_MVT_v2i32_r(RetVT, Op0);
769 case MVT::v4i32: return fastEmit_ARMISD_VREV64_MVT_v4i32_r(RetVT, Op0);
770 case MVT::v4f16: return fastEmit_ARMISD_VREV64_MVT_v4f16_r(RetVT, Op0);
771 case MVT::v8f16: return fastEmit_ARMISD_VREV64_MVT_v8f16_r(RetVT, Op0);
772 case MVT::v4bf16: return fastEmit_ARMISD_VREV64_MVT_v4bf16_r(RetVT, Op0);
773 case MVT::v8bf16: return fastEmit_ARMISD_VREV64_MVT_v8bf16_r(RetVT, Op0);
774 case MVT::v2f32: return fastEmit_ARMISD_VREV64_MVT_v2f32_r(RetVT, Op0);
775 case MVT::v4f32: return fastEmit_ARMISD_VREV64_MVT_v4f32_r(RetVT, Op0);
776 default: return Register();
777 }
778}
779
780// FastEmit functions for ARMISD::WIN__DBZCHK.
781
782Register fastEmit_ARMISD_WIN__DBZCHK_MVT_i32_r(MVT RetVT, Register Op0) {
783 if (RetVT.SimpleTy != MVT::isVoid)
784 return Register();
785 return fastEmitInst_r(MachineInstOpcode: ARM::WIN__DBZCHK, RC: &ARM::tGPRRegClass, Op0);
786}
787
788Register fastEmit_ARMISD_WIN__DBZCHK_r(MVT VT, MVT RetVT, Register Op0) {
789 switch (VT.SimpleTy) {
790 case MVT::i32: return fastEmit_ARMISD_WIN__DBZCHK_MVT_i32_r(RetVT, Op0);
791 default: return Register();
792 }
793}
794
795// FastEmit functions for ARMISD::WLSSETUP.
796
797Register fastEmit_ARMISD_WLSSETUP_MVT_i32_r(MVT RetVT, Register Op0) {
798 if (RetVT.SimpleTy != MVT::i32)
799 return Register();
800 if ((Subtarget->hasLOB()) && (Subtarget->hasV8_1MMainlineOps()) && (Subtarget->isThumb2())) {
801 return fastEmitInst_r(MachineInstOpcode: ARM::t2WhileLoopSetup, RC: &ARM::GPRlrRegClass, Op0);
802 }
803 return Register();
804}
805
806Register fastEmit_ARMISD_WLSSETUP_r(MVT VT, MVT RetVT, Register Op0) {
807 switch (VT.SimpleTy) {
808 case MVT::i32: return fastEmit_ARMISD_WLSSETUP_MVT_i32_r(RetVT, Op0);
809 default: return Register();
810 }
811}
812
813// FastEmit functions for ARMISD::tSECALL.
814
815Register fastEmit_ARMISD_tSECALL_MVT_i32_r(MVT RetVT, Register Op0) {
816 if (RetVT.SimpleTy != MVT::isVoid)
817 return Register();
818 if ((Subtarget->has8MSecExt()) && (Subtarget->isThumb())) {
819 return fastEmitInst_r(MachineInstOpcode: ARM::tBLXNS_CALL, RC: &ARM::GPRnopcRegClass, Op0);
820 }
821 return Register();
822}
823
824Register fastEmit_ARMISD_tSECALL_r(MVT VT, MVT RetVT, Register Op0) {
825 switch (VT.SimpleTy) {
826 case MVT::i32: return fastEmit_ARMISD_tSECALL_MVT_i32_r(RetVT, Op0);
827 default: return Register();
828 }
829}
830
831// FastEmit functions for ISD::ABS.
832
833Register fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, Register Op0) {
834 if (RetVT.SimpleTy != MVT::v8i8)
835 return Register();
836 if ((Subtarget->hasNEON())) {
837 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv8i8, RC: &ARM::DPRRegClass, Op0);
838 }
839 return Register();
840}
841
842Register fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, Register Op0) {
843 if (RetVT.SimpleTy != MVT::v16i8)
844 return Register();
845 if ((Subtarget->hasMVEIntegerOps())) {
846 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSs8, RC: &ARM::MQPRRegClass, Op0);
847 }
848 if ((Subtarget->hasNEON())) {
849 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv16i8, RC: &ARM::QPRRegClass, Op0);
850 }
851 return Register();
852}
853
854Register fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, Register Op0) {
855 if (RetVT.SimpleTy != MVT::v4i16)
856 return Register();
857 if ((Subtarget->hasNEON())) {
858 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv4i16, RC: &ARM::DPRRegClass, Op0);
859 }
860 return Register();
861}
862
863Register fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, Register Op0) {
864 if (RetVT.SimpleTy != MVT::v8i16)
865 return Register();
866 if ((Subtarget->hasMVEIntegerOps())) {
867 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSs16, RC: &ARM::MQPRRegClass, Op0);
868 }
869 if ((Subtarget->hasNEON())) {
870 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv8i16, RC: &ARM::QPRRegClass, Op0);
871 }
872 return Register();
873}
874
875Register fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, Register Op0) {
876 if (RetVT.SimpleTy != MVT::v2i32)
877 return Register();
878 if ((Subtarget->hasNEON())) {
879 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv2i32, RC: &ARM::DPRRegClass, Op0);
880 }
881 return Register();
882}
883
884Register fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, Register Op0) {
885 if (RetVT.SimpleTy != MVT::v4i32)
886 return Register();
887 if ((Subtarget->hasMVEIntegerOps())) {
888 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSs32, RC: &ARM::MQPRRegClass, Op0);
889 }
890 if ((Subtarget->hasNEON())) {
891 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv4i32, RC: &ARM::QPRRegClass, Op0);
892 }
893 return Register();
894}
895
896Register fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, Register Op0) {
897 switch (VT.SimpleTy) {
898 case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0);
899 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
900 case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0);
901 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
902 case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0);
903 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
904 default: return Register();
905 }
906}
907
908// FastEmit functions for ISD::ANY_EXTEND.
909
910Register fastEmit_ISD_ANY_EXTEND_MVT_v8i8_r(MVT RetVT, Register Op0) {
911 if (RetVT.SimpleTy != MVT::v8i16)
912 return Register();
913 if ((Subtarget->hasNEON())) {
914 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv8i16, RC: &ARM::QPRRegClass, Op0);
915 }
916 return Register();
917}
918
919Register fastEmit_ISD_ANY_EXTEND_MVT_v4i16_r(MVT RetVT, Register Op0) {
920 if (RetVT.SimpleTy != MVT::v4i32)
921 return Register();
922 if ((Subtarget->hasNEON())) {
923 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv4i32, RC: &ARM::QPRRegClass, Op0);
924 }
925 return Register();
926}
927
928Register fastEmit_ISD_ANY_EXTEND_MVT_v2i32_r(MVT RetVT, Register Op0) {
929 if (RetVT.SimpleTy != MVT::v2i64)
930 return Register();
931 if ((Subtarget->hasNEON())) {
932 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv2i64, RC: &ARM::QPRRegClass, Op0);
933 }
934 return Register();
935}
936
937Register fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
938 switch (VT.SimpleTy) {
939 case MVT::v8i8: return fastEmit_ISD_ANY_EXTEND_MVT_v8i8_r(RetVT, Op0);
940 case MVT::v4i16: return fastEmit_ISD_ANY_EXTEND_MVT_v4i16_r(RetVT, Op0);
941 case MVT::v2i32: return fastEmit_ISD_ANY_EXTEND_MVT_v2i32_r(RetVT, Op0);
942 default: return Register();
943 }
944}
945
946// FastEmit functions for ISD::BITCAST.
947
948Register fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, Register Op0) {
949 if (RetVT.SimpleTy != MVT::f32)
950 return Register();
951 if ((Subtarget->hasFPRegs()) && (Subtarget->preferVMOVSR() ||!Subtarget->useNEONForSinglePrecisionFP())) {
952 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVSR, RC: &ARM::SPRRegClass, Op0);
953 }
954 return Register();
955}
956
957Register fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, Register Op0) {
958 if (RetVT.SimpleTy != MVT::i32)
959 return Register();
960 if ((Subtarget->hasFPRegs())) {
961 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVRS, RC: &ARM::GPRRegClass, Op0);
962 }
963 return Register();
964}
965
966Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Register Op0) {
967 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
968 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
969 }
970 return Register();
971}
972
973Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Register Op0) {
974 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
975 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
976 }
977 return Register();
978}
979
980Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Register Op0) {
981 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
982 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
983 }
984 return Register();
985}
986
987Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Register Op0) {
988 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
989 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
990 }
991 return Register();
992}
993
994Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Register Op0) {
995 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
996 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
997 }
998 return Register();
999}
1000
1001Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Register Op0) {
1002 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1003 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1004 }
1005 return Register();
1006}
1007
1008Register fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, Register Op0) {
1009switch (RetVT.SimpleTy) {
1010 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0);
1011 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0);
1012 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0);
1013 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0);
1014 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Op0);
1015 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0);
1016 default: return Register();
1017}
1018}
1019
1020Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Register Op0) {
1021 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1022 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
1023 }
1024 return Register();
1025}
1026
1027Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Register Op0) {
1028 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1029 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1030 }
1031 return Register();
1032}
1033
1034Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Register Op0) {
1035 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1036 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1037 }
1038 return Register();
1039}
1040
1041Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Register Op0) {
1042 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1043 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
1044 }
1045 return Register();
1046}
1047
1048Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Register Op0) {
1049 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1050 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1051 }
1052 return Register();
1053}
1054
1055Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Register Op0) {
1056 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1057 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1058 }
1059 return Register();
1060}
1061
1062Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Register Op0) {
1063 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1064 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1065 }
1066 return Register();
1067}
1068
1069Register fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, Register Op0) {
1070switch (RetVT.SimpleTy) {
1071 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0);
1072 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0);
1073 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0);
1074 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0);
1075 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0);
1076 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Op0);
1077 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0);
1078 default: return Register();
1079}
1080}
1081
1082Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Register Op0) {
1083 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1084 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1085 }
1086 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1087 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1088 }
1089 return Register();
1090}
1091
1092Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Register Op0) {
1093 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1094 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1095 }
1096 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1097 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1098 }
1099 return Register();
1100}
1101
1102Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Register Op0) {
1103 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1104 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1105 }
1106 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1107 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1108 }
1109 return Register();
1110}
1111
1112Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Register Op0) {
1113 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1114 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1115 }
1116 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1117 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1118 }
1119 return Register();
1120}
1121
1122Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Register Op0) {
1123 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1124 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1125 }
1126 return Register();
1127}
1128
1129Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Register Op0) {
1130 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1131 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1132 }
1133 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1134 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1135 }
1136 return Register();
1137}
1138
1139Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Register Op0) {
1140 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1141 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1142 }
1143 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1144 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1145 }
1146 return Register();
1147}
1148
1149Register fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, Register Op0) {
1150switch (RetVT.SimpleTy) {
1151 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0);
1152 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0);
1153 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0);
1154 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0);
1155 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Op0);
1156 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0);
1157 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0);
1158 default: return Register();
1159}
1160}
1161
1162Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Register Op0) {
1163 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1164 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1165 }
1166 return Register();
1167}
1168
1169Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Register Op0) {
1170 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1171 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1172 }
1173 return Register();
1174}
1175
1176Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Register Op0) {
1177 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1178 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1179 }
1180 return Register();
1181}
1182
1183Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Register Op0) {
1184 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1185 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1186 }
1187 return Register();
1188}
1189
1190Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Register Op0) {
1191 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1192 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1193 }
1194 return Register();
1195}
1196
1197Register fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, Register Op0) {
1198switch (RetVT.SimpleTy) {
1199 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0);
1200 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0);
1201 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0);
1202 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0);
1203 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0);
1204 default: return Register();
1205}
1206}
1207
1208Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Register Op0) {
1209 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1210 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1211 }
1212 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1213 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1214 }
1215 return Register();
1216}
1217
1218Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Register Op0) {
1219 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1220 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1221 }
1222 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1223 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1224 }
1225 return Register();
1226}
1227
1228Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Register Op0) {
1229 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1230 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1231 }
1232 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1233 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1234 }
1235 return Register();
1236}
1237
1238Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Register Op0) {
1239 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1240 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1241 }
1242 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1243 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1244 }
1245 return Register();
1246}
1247
1248Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Register Op0) {
1249 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1250 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1251 }
1252 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1253 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1254 }
1255 return Register();
1256}
1257
1258Register fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, Register Op0) {
1259switch (RetVT.SimpleTy) {
1260 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0);
1261 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0);
1262 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0);
1263 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0);
1264 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0);
1265 default: return Register();
1266}
1267}
1268
1269Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Register Op0) {
1270 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1271 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1272 }
1273 return Register();
1274}
1275
1276Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Register Op0) {
1277 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1278 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1279 }
1280 return Register();
1281}
1282
1283Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Register Op0) {
1284 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1285 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1286 }
1287 return Register();
1288}
1289
1290Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Register Op0) {
1291 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1292 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1293 }
1294 return Register();
1295}
1296
1297Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Register Op0) {
1298 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1299 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1300 }
1301 return Register();
1302}
1303
1304Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Register Op0) {
1305 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1306 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1307 }
1308 return Register();
1309}
1310
1311Register fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, Register Op0) {
1312switch (RetVT.SimpleTy) {
1313 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0);
1314 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0);
1315 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0);
1316 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0);
1317 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0);
1318 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Op0);
1319 default: return Register();
1320}
1321}
1322
1323Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Register Op0) {
1324 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1325 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1326 }
1327 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1328 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1329 }
1330 return Register();
1331}
1332
1333Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Register Op0) {
1334 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1335 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1336 }
1337 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1338 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1339 }
1340 return Register();
1341}
1342
1343Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Register Op0) {
1344 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1345 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1346 }
1347 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1348 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1349 }
1350 return Register();
1351}
1352
1353Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Register Op0) {
1354 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1355 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1356 }
1357 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1358 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1359 }
1360 return Register();
1361}
1362
1363Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Register Op0) {
1364 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1365 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1366 }
1367 return Register();
1368}
1369
1370Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Register Op0) {
1371 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1372 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1373 }
1374 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1375 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1376 }
1377 return Register();
1378}
1379
1380Register fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, Register Op0) {
1381switch (RetVT.SimpleTy) {
1382 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0);
1383 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0);
1384 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0);
1385 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0);
1386 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Op0);
1387 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0);
1388 default: return Register();
1389}
1390}
1391
1392Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Register Op0) {
1393 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1394 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
1395 }
1396 return Register();
1397}
1398
1399Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Register Op0) {
1400 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1401 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1402 }
1403 return Register();
1404}
1405
1406Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Register Op0) {
1407 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1408 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1409 }
1410 return Register();
1411}
1412
1413Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Register Op0) {
1414 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1415 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1416 }
1417 return Register();
1418}
1419
1420Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Register Op0) {
1421 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1422 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1423 }
1424 return Register();
1425}
1426
1427Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Register Op0) {
1428 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1429 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1430 }
1431 return Register();
1432}
1433
1434Register fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, Register Op0) {
1435switch (RetVT.SimpleTy) {
1436 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0);
1437 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0);
1438 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0);
1439 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0);
1440 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Op0);
1441 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0);
1442 default: return Register();
1443}
1444}
1445
1446Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Register Op0) {
1447 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1448 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1449 }
1450 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1451 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1452 }
1453 return Register();
1454}
1455
1456Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Register Op0) {
1457 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1458 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1459 }
1460 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1461 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1462 }
1463 return Register();
1464}
1465
1466Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Register Op0) {
1467 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1468 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1469 }
1470 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1471 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1472 }
1473 return Register();
1474}
1475
1476Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Register Op0) {
1477 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1478 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1479 }
1480 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1481 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1482 }
1483 return Register();
1484}
1485
1486Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Register Op0) {
1487 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1488 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1489 }
1490 return Register();
1491}
1492
1493Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Register Op0) {
1494 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1495 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1496 }
1497 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1498 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1499 }
1500 return Register();
1501}
1502
1503Register fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, Register Op0) {
1504switch (RetVT.SimpleTy) {
1505 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0);
1506 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0);
1507 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0);
1508 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0);
1509 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Op0);
1510 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0);
1511 default: return Register();
1512}
1513}
1514
1515Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Register Op0) {
1516 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1517 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1518 }
1519 return Register();
1520}
1521
1522Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Register Op0) {
1523 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1524 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1525 }
1526 return Register();
1527}
1528
1529Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Register Op0) {
1530 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1531 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1532 }
1533 return Register();
1534}
1535
1536Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Register Op0) {
1537 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1538 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1539 }
1540 return Register();
1541}
1542
1543Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Register Op0) {
1544 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1545 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1546 }
1547 return Register();
1548}
1549
1550Register fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, Register Op0) {
1551switch (RetVT.SimpleTy) {
1552 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0);
1553 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0);
1554 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0);
1555 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0);
1556 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0);
1557 default: return Register();
1558}
1559}
1560
1561Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Register Op0) {
1562 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1563 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1564 }
1565 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1566 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1567 }
1568 return Register();
1569}
1570
1571Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Register Op0) {
1572 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1573 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1574 }
1575 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1576 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1577 }
1578 return Register();
1579}
1580
1581Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Register Op0) {
1582 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1583 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1584 }
1585 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1586 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1587 }
1588 return Register();
1589}
1590
1591Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Register Op0) {
1592 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1593 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1594 }
1595 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1596 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1597 }
1598 return Register();
1599}
1600
1601Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Register Op0) {
1602 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1603 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1604 }
1605 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1606 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1607 }
1608 return Register();
1609}
1610
1611Register fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, Register Op0) {
1612switch (RetVT.SimpleTy) {
1613 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0);
1614 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0);
1615 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0);
1616 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0);
1617 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0);
1618 default: return Register();
1619}
1620}
1621
1622Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Register Op0) {
1623 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1624 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1625 }
1626 return Register();
1627}
1628
1629Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Register Op0) {
1630 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1631 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1632 }
1633 return Register();
1634}
1635
1636Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Register Op0) {
1637 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1638 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1639 }
1640 return Register();
1641}
1642
1643Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Register Op0) {
1644 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1645 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1646 }
1647 return Register();
1648}
1649
1650Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Register Op0) {
1651 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1652 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1653 }
1654 return Register();
1655}
1656
1657Register fastEmit_ISD_BITCAST_MVT_v4bf16_r(MVT RetVT, Register Op0) {
1658switch (RetVT.SimpleTy) {
1659 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Op0);
1660 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Op0);
1661 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Op0);
1662 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Op0);
1663 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Op0);
1664 default: return Register();
1665}
1666}
1667
1668Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Register Op0) {
1669 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1670 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1671 }
1672 return Register();
1673}
1674
1675Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Register Op0) {
1676 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1677 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1678 }
1679 return Register();
1680}
1681
1682Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Register Op0) {
1683 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1684 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1685 }
1686 return Register();
1687}
1688
1689Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Register Op0) {
1690 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1691 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1692 }
1693 return Register();
1694}
1695
1696Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Register Op0) {
1697 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1698 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1699 }
1700 return Register();
1701}
1702
1703Register fastEmit_ISD_BITCAST_MVT_v8bf16_r(MVT RetVT, Register Op0) {
1704switch (RetVT.SimpleTy) {
1705 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Op0);
1706 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Op0);
1707 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Op0);
1708 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Op0);
1709 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Op0);
1710 default: return Register();
1711}
1712}
1713
1714Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Register Op0) {
1715 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1716 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1717 }
1718 return Register();
1719}
1720
1721Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Register Op0) {
1722 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1723 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1724 }
1725 return Register();
1726}
1727
1728Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Register Op0) {
1729 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1730 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1731 }
1732 return Register();
1733}
1734
1735Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Register Op0) {
1736 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1737 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1738 }
1739 return Register();
1740}
1741
1742Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Register Op0) {
1743 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1744 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1745 }
1746 return Register();
1747}
1748
1749Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Register Op0) {
1750 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1751 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1752 }
1753 return Register();
1754}
1755
1756Register fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, Register Op0) {
1757switch (RetVT.SimpleTy) {
1758 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0);
1759 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0);
1760 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0);
1761 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0);
1762 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0);
1763 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Op0);
1764 default: return Register();
1765}
1766}
1767
1768Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Register Op0) {
1769 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1770 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1771 }
1772 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1773 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1774 }
1775 return Register();
1776}
1777
1778Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Register Op0) {
1779 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1780 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1781 }
1782 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1783 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1784 }
1785 return Register();
1786}
1787
1788Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Register Op0) {
1789 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1790 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1791 }
1792 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1793 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1794 }
1795 return Register();
1796}
1797
1798Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Register Op0) {
1799 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1800 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1801 }
1802 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1803 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1804 }
1805 return Register();
1806}
1807
1808Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Register Op0) {
1809 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1810 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1811 }
1812 return Register();
1813}
1814
1815Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Register Op0) {
1816 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1817 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1818 }
1819 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1820 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1821 }
1822 return Register();
1823}
1824
1825Register fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, Register Op0) {
1826switch (RetVT.SimpleTy) {
1827 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0);
1828 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0);
1829 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0);
1830 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0);
1831 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Op0);
1832 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0);
1833 default: return Register();
1834}
1835}
1836
1837Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Register Op0) {
1838 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1839 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1840 }
1841 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1842 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1843 }
1844 return Register();
1845}
1846
1847Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Register Op0) {
1848 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1849 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1850 }
1851 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1852 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1853 }
1854 return Register();
1855}
1856
1857Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Register Op0) {
1858 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1859 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1860 }
1861 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1862 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1863 }
1864 return Register();
1865}
1866
1867Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Register Op0) {
1868 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1869 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1870 }
1871 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1872 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1873 }
1874 return Register();
1875}
1876
1877Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Register Op0) {
1878 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1879 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1880 }
1881 return Register();
1882}
1883
1884Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Register Op0) {
1885 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1886 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1887 }
1888 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1889 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1890 }
1891 return Register();
1892}
1893
1894Register fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, Register Op0) {
1895switch (RetVT.SimpleTy) {
1896 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0);
1897 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0);
1898 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0);
1899 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0);
1900 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Op0);
1901 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0);
1902 default: return Register();
1903}
1904}
1905
1906Register fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, Register Op0) {
1907 switch (VT.SimpleTy) {
1908 case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0);
1909 case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0);
1910 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
1911 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0);
1912 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0);
1913 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0);
1914 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0);
1915 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0);
1916 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0);
1917 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0);
1918 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0);
1919 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0);
1920 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0);
1921 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v4bf16_r(RetVT, Op0);
1922 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v8bf16_r(RetVT, Op0);
1923 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0);
1924 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0);
1925 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0);
1926 default: return Register();
1927 }
1928}
1929
1930// FastEmit functions for ISD::BITREVERSE.
1931
1932Register fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, Register Op0) {
1933 if (RetVT.SimpleTy != MVT::i32)
1934 return Register();
1935 if ((Subtarget->isThumb2())) {
1936 return fastEmitInst_r(MachineInstOpcode: ARM::t2RBIT, RC: &ARM::rGPRRegClass, Op0);
1937 }
1938 if ((Subtarget->hasV6T2Ops()) && (!Subtarget->isThumb())) {
1939 return fastEmitInst_r(MachineInstOpcode: ARM::RBIT, RC: &ARM::GPRRegClass, Op0);
1940 }
1941 return Register();
1942}
1943
1944Register fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, Register Op0) {
1945 switch (VT.SimpleTy) {
1946 case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0);
1947 default: return Register();
1948 }
1949}
1950
1951// FastEmit functions for ISD::BRIND.
1952
1953Register fastEmit_ISD_BRIND_MVT_i32_r(MVT RetVT, Register Op0) {
1954 if (RetVT.SimpleTy != MVT::isVoid)
1955 return Register();
1956 if ((Subtarget->isThumb())) {
1957 return fastEmitInst_r(MachineInstOpcode: ARM::tBRIND, RC: &ARM::GPRRegClass, Op0);
1958 }
1959 if ((!Subtarget->isThumb()) && (!Subtarget->hasV4TOps())) {
1960 return fastEmitInst_r(MachineInstOpcode: ARM::MOVPCRX, RC: &ARM::GPRRegClass, Op0);
1961 }
1962 if ((Subtarget->hasV4TOps()) && (!Subtarget->isThumb())) {
1963 return fastEmitInst_r(MachineInstOpcode: ARM::BX, RC: &ARM::GPRRegClass, Op0);
1964 }
1965 return Register();
1966}
1967
1968Register fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, Register Op0) {
1969 switch (VT.SimpleTy) {
1970 case MVT::i32: return fastEmit_ISD_BRIND_MVT_i32_r(RetVT, Op0);
1971 default: return Register();
1972 }
1973}
1974
1975// FastEmit functions for ISD::BSWAP.
1976
1977Register fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, Register Op0) {
1978 if (RetVT.SimpleTy != MVT::i32)
1979 return Register();
1980 if ((Subtarget->isThumb2())) {
1981 return fastEmitInst_r(MachineInstOpcode: ARM::t2REV, RC: &ARM::rGPRRegClass, Op0);
1982 }
1983 if ((Subtarget->hasV6Ops()) && (Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
1984 return fastEmitInst_r(MachineInstOpcode: ARM::tREV, RC: &ARM::tGPRRegClass, Op0);
1985 }
1986 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
1987 return fastEmitInst_r(MachineInstOpcode: ARM::REV, RC: &ARM::GPRRegClass, Op0);
1988 }
1989 return Register();
1990}
1991
1992Register fastEmit_ISD_BSWAP_MVT_v8i16_r(MVT RetVT, Register Op0) {
1993 if (RetVT.SimpleTy != MVT::v8i16)
1994 return Register();
1995 if ((Subtarget->hasMVEIntegerOps())) {
1996 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1997 }
1998 return Register();
1999}
2000
2001Register fastEmit_ISD_BSWAP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2002 if (RetVT.SimpleTy != MVT::v4i32)
2003 return Register();
2004 if ((Subtarget->hasMVEIntegerOps())) {
2005 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
2006 }
2007 return Register();
2008}
2009
2010Register fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, Register Op0) {
2011 switch (VT.SimpleTy) {
2012 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
2013 case MVT::v8i16: return fastEmit_ISD_BSWAP_MVT_v8i16_r(RetVT, Op0);
2014 case MVT::v4i32: return fastEmit_ISD_BSWAP_MVT_v4i32_r(RetVT, Op0);
2015 default: return Register();
2016 }
2017}
2018
2019// FastEmit functions for ISD::CTLS.
2020
2021Register fastEmit_ISD_CTLS_MVT_v8i8_r(MVT RetVT, Register Op0) {
2022 if (RetVT.SimpleTy != MVT::v8i8)
2023 return Register();
2024 if ((Subtarget->hasNEON())) {
2025 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv8i8, RC: &ARM::DPRRegClass, Op0);
2026 }
2027 return Register();
2028}
2029
2030Register fastEmit_ISD_CTLS_MVT_v16i8_r(MVT RetVT, Register Op0) {
2031 if (RetVT.SimpleTy != MVT::v16i8)
2032 return Register();
2033 if ((Subtarget->hasMVEIntegerOps())) {
2034 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLSs8, RC: &ARM::MQPRRegClass, Op0);
2035 }
2036 if ((Subtarget->hasNEON())) {
2037 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv16i8, RC: &ARM::QPRRegClass, Op0);
2038 }
2039 return Register();
2040}
2041
2042Register fastEmit_ISD_CTLS_MVT_v4i16_r(MVT RetVT, Register Op0) {
2043 if (RetVT.SimpleTy != MVT::v4i16)
2044 return Register();
2045 if ((Subtarget->hasNEON())) {
2046 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv4i16, RC: &ARM::DPRRegClass, Op0);
2047 }
2048 return Register();
2049}
2050
2051Register fastEmit_ISD_CTLS_MVT_v8i16_r(MVT RetVT, Register Op0) {
2052 if (RetVT.SimpleTy != MVT::v8i16)
2053 return Register();
2054 if ((Subtarget->hasMVEIntegerOps())) {
2055 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLSs16, RC: &ARM::MQPRRegClass, Op0);
2056 }
2057 if ((Subtarget->hasNEON())) {
2058 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv8i16, RC: &ARM::QPRRegClass, Op0);
2059 }
2060 return Register();
2061}
2062
2063Register fastEmit_ISD_CTLS_MVT_v2i32_r(MVT RetVT, Register Op0) {
2064 if (RetVT.SimpleTy != MVT::v2i32)
2065 return Register();
2066 if ((Subtarget->hasNEON())) {
2067 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv2i32, RC: &ARM::DPRRegClass, Op0);
2068 }
2069 return Register();
2070}
2071
2072Register fastEmit_ISD_CTLS_MVT_v4i32_r(MVT RetVT, Register Op0) {
2073 if (RetVT.SimpleTy != MVT::v4i32)
2074 return Register();
2075 if ((Subtarget->hasMVEIntegerOps())) {
2076 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLSs32, RC: &ARM::MQPRRegClass, Op0);
2077 }
2078 if ((Subtarget->hasNEON())) {
2079 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv4i32, RC: &ARM::QPRRegClass, Op0);
2080 }
2081 return Register();
2082}
2083
2084Register fastEmit_ISD_CTLS_r(MVT VT, MVT RetVT, Register Op0) {
2085 switch (VT.SimpleTy) {
2086 case MVT::v8i8: return fastEmit_ISD_CTLS_MVT_v8i8_r(RetVT, Op0);
2087 case MVT::v16i8: return fastEmit_ISD_CTLS_MVT_v16i8_r(RetVT, Op0);
2088 case MVT::v4i16: return fastEmit_ISD_CTLS_MVT_v4i16_r(RetVT, Op0);
2089 case MVT::v8i16: return fastEmit_ISD_CTLS_MVT_v8i16_r(RetVT, Op0);
2090 case MVT::v2i32: return fastEmit_ISD_CTLS_MVT_v2i32_r(RetVT, Op0);
2091 case MVT::v4i32: return fastEmit_ISD_CTLS_MVT_v4i32_r(RetVT, Op0);
2092 default: return Register();
2093 }
2094}
2095
2096// FastEmit functions for ISD::CTLZ.
2097
2098Register fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, Register Op0) {
2099 if (RetVT.SimpleTy != MVT::i32)
2100 return Register();
2101 if ((Subtarget->isThumb2())) {
2102 return fastEmitInst_r(MachineInstOpcode: ARM::t2CLZ, RC: &ARM::rGPRRegClass, Op0);
2103 }
2104 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb())) {
2105 return fastEmitInst_r(MachineInstOpcode: ARM::CLZ, RC: &ARM::GPRRegClass, Op0);
2106 }
2107 return Register();
2108}
2109
2110Register fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, Register Op0) {
2111 if (RetVT.SimpleTy != MVT::v8i8)
2112 return Register();
2113 if ((Subtarget->hasNEON())) {
2114 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv8i8, RC: &ARM::DPRRegClass, Op0);
2115 }
2116 return Register();
2117}
2118
2119Register fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, Register Op0) {
2120 if (RetVT.SimpleTy != MVT::v16i8)
2121 return Register();
2122 if ((Subtarget->hasMVEIntegerOps())) {
2123 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLZs8, RC: &ARM::MQPRRegClass, Op0);
2124 }
2125 if ((Subtarget->hasNEON())) {
2126 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv16i8, RC: &ARM::QPRRegClass, Op0);
2127 }
2128 return Register();
2129}
2130
2131Register fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, Register Op0) {
2132 if (RetVT.SimpleTy != MVT::v4i16)
2133 return Register();
2134 if ((Subtarget->hasNEON())) {
2135 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv4i16, RC: &ARM::DPRRegClass, Op0);
2136 }
2137 return Register();
2138}
2139
2140Register fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, Register Op0) {
2141 if (RetVT.SimpleTy != MVT::v8i16)
2142 return Register();
2143 if ((Subtarget->hasMVEIntegerOps())) {
2144 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLZs16, RC: &ARM::MQPRRegClass, Op0);
2145 }
2146 if ((Subtarget->hasNEON())) {
2147 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv8i16, RC: &ARM::QPRRegClass, Op0);
2148 }
2149 return Register();
2150}
2151
2152Register fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, Register Op0) {
2153 if (RetVT.SimpleTy != MVT::v2i32)
2154 return Register();
2155 if ((Subtarget->hasNEON())) {
2156 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv2i32, RC: &ARM::DPRRegClass, Op0);
2157 }
2158 return Register();
2159}
2160
2161Register fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, Register Op0) {
2162 if (RetVT.SimpleTy != MVT::v4i32)
2163 return Register();
2164 if ((Subtarget->hasMVEIntegerOps())) {
2165 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLZs32, RC: &ARM::MQPRRegClass, Op0);
2166 }
2167 if ((Subtarget->hasNEON())) {
2168 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv4i32, RC: &ARM::QPRRegClass, Op0);
2169 }
2170 return Register();
2171}
2172
2173Register fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, Register Op0) {
2174 switch (VT.SimpleTy) {
2175 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
2176 case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0);
2177 case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0);
2178 case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0);
2179 case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0);
2180 case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0);
2181 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
2182 default: return Register();
2183 }
2184}
2185
2186// FastEmit functions for ISD::CTPOP.
2187
2188Register fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, Register Op0) {
2189 if (RetVT.SimpleTy != MVT::v8i8)
2190 return Register();
2191 if ((Subtarget->hasNEON())) {
2192 return fastEmitInst_r(MachineInstOpcode: ARM::VCNTd, RC: &ARM::DPRRegClass, Op0);
2193 }
2194 return Register();
2195}
2196
2197Register fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, Register Op0) {
2198 if (RetVT.SimpleTy != MVT::v16i8)
2199 return Register();
2200 if ((Subtarget->hasNEON())) {
2201 return fastEmitInst_r(MachineInstOpcode: ARM::VCNTq, RC: &ARM::QPRRegClass, Op0);
2202 }
2203 return Register();
2204}
2205
2206Register fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, Register Op0) {
2207 switch (VT.SimpleTy) {
2208 case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0);
2209 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
2210 default: return Register();
2211 }
2212}
2213
2214// FastEmit functions for ISD::FABS.
2215
2216Register fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, Register Op0) {
2217 if (RetVT.SimpleTy != MVT::f16)
2218 return Register();
2219 if ((Subtarget->hasFullFP16())) {
2220 return fastEmitInst_r(MachineInstOpcode: ARM::VABSH, RC: &ARM::HPRRegClass, Op0);
2221 }
2222 return Register();
2223}
2224
2225Register fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, Register Op0) {
2226 if (RetVT.SimpleTy != MVT::f32)
2227 return Register();
2228 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
2229 return fastEmitInst_r(MachineInstOpcode: ARM::VABSS, RC: &ARM::SPRRegClass, Op0);
2230 }
2231 return Register();
2232}
2233
2234Register fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, Register Op0) {
2235 if (RetVT.SimpleTy != MVT::f64)
2236 return Register();
2237 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2238 return fastEmitInst_r(MachineInstOpcode: ARM::VABSD, RC: &ARM::DPRRegClass, Op0);
2239 }
2240 return Register();
2241}
2242
2243Register fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, Register Op0) {
2244 if (RetVT.SimpleTy != MVT::v4f16)
2245 return Register();
2246 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2247 return fastEmitInst_r(MachineInstOpcode: ARM::VABShd, RC: &ARM::DPRRegClass, Op0);
2248 }
2249 return Register();
2250}
2251
2252Register fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, Register Op0) {
2253 if (RetVT.SimpleTy != MVT::v8f16)
2254 return Register();
2255 if ((Subtarget->hasMVEIntegerOps())) {
2256 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSf16, RC: &ARM::MQPRRegClass, Op0);
2257 }
2258 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2259 return fastEmitInst_r(MachineInstOpcode: ARM::VABShq, RC: &ARM::QPRRegClass, Op0);
2260 }
2261 return Register();
2262}
2263
2264Register fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, Register Op0) {
2265 if (RetVT.SimpleTy != MVT::v2f32)
2266 return Register();
2267 if ((Subtarget->hasNEON())) {
2268 return fastEmitInst_r(MachineInstOpcode: ARM::VABSfd, RC: &ARM::DPRRegClass, Op0);
2269 }
2270 return Register();
2271}
2272
2273Register fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, Register Op0) {
2274 if (RetVT.SimpleTy != MVT::v4f32)
2275 return Register();
2276 if ((Subtarget->hasMVEIntegerOps())) {
2277 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSf32, RC: &ARM::MQPRRegClass, Op0);
2278 }
2279 if ((Subtarget->hasNEON())) {
2280 return fastEmitInst_r(MachineInstOpcode: ARM::VABSfq, RC: &ARM::QPRRegClass, Op0);
2281 }
2282 return Register();
2283}
2284
2285Register fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, Register Op0) {
2286 switch (VT.SimpleTy) {
2287 case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0);
2288 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
2289 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
2290 case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0);
2291 case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0);
2292 case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0);
2293 case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0);
2294 default: return Register();
2295 }
2296}
2297
2298// FastEmit functions for ISD::FCEIL.
2299
2300Register fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
2301 if (RetVT.SimpleTy != MVT::f16)
2302 return Register();
2303 if ((Subtarget->hasFullFP16())) {
2304 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPH, RC: &ARM::HPRRegClass, Op0);
2305 }
2306 return Register();
2307}
2308
2309Register fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
2310 if (RetVT.SimpleTy != MVT::f32)
2311 return Register();
2312 if ((Subtarget->hasFPARMv8Base())) {
2313 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPS, RC: &ARM::SPRRegClass, Op0);
2314 }
2315 return Register();
2316}
2317
2318Register fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
2319 if (RetVT.SimpleTy != MVT::f64)
2320 return Register();
2321 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2322 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPD, RC: &ARM::DPRRegClass, Op0);
2323 }
2324 return Register();
2325}
2326
2327Register fastEmit_ISD_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
2328 if (RetVT.SimpleTy != MVT::v4f16)
2329 return Register();
2330 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2331 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNDh, RC: &ARM::DPRRegClass, Op0);
2332 }
2333 return Register();
2334}
2335
2336Register fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
2337 if (RetVT.SimpleTy != MVT::v8f16)
2338 return Register();
2339 if ((Subtarget->hasMVEFloatOps())) {
2340 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16P, RC: &ARM::MQPRRegClass, Op0);
2341 }
2342 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2343 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNQh, RC: &ARM::QPRRegClass, Op0);
2344 }
2345 return Register();
2346}
2347
2348Register fastEmit_ISD_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
2349 if (RetVT.SimpleTy != MVT::v2f32)
2350 return Register();
2351 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2352 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNDf, RC: &ARM::DPRRegClass, Op0);
2353 }
2354 return Register();
2355}
2356
2357Register fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
2358 if (RetVT.SimpleTy != MVT::v4f32)
2359 return Register();
2360 if ((Subtarget->hasMVEFloatOps())) {
2361 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32P, RC: &ARM::MQPRRegClass, Op0);
2362 }
2363 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2364 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNQf, RC: &ARM::QPRRegClass, Op0);
2365 }
2366 return Register();
2367}
2368
2369Register fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
2370 switch (VT.SimpleTy) {
2371 case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0);
2372 case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0);
2373 case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0);
2374 case MVT::v4f16: return fastEmit_ISD_FCEIL_MVT_v4f16_r(RetVT, Op0);
2375 case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0);
2376 case MVT::v2f32: return fastEmit_ISD_FCEIL_MVT_v2f32_r(RetVT, Op0);
2377 case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0);
2378 default: return Register();
2379 }
2380}
2381
2382// FastEmit functions for ISD::FFLOOR.
2383
2384Register fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
2385 if (RetVT.SimpleTy != MVT::f16)
2386 return Register();
2387 if ((Subtarget->hasFullFP16())) {
2388 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMH, RC: &ARM::HPRRegClass, Op0);
2389 }
2390 return Register();
2391}
2392
2393Register fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
2394 if (RetVT.SimpleTy != MVT::f32)
2395 return Register();
2396 if ((Subtarget->hasFPARMv8Base())) {
2397 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMS, RC: &ARM::SPRRegClass, Op0);
2398 }
2399 return Register();
2400}
2401
2402Register fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
2403 if (RetVT.SimpleTy != MVT::f64)
2404 return Register();
2405 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2406 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMD, RC: &ARM::DPRRegClass, Op0);
2407 }
2408 return Register();
2409}
2410
2411Register fastEmit_ISD_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
2412 if (RetVT.SimpleTy != MVT::v4f16)
2413 return Register();
2414 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2415 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNDh, RC: &ARM::DPRRegClass, Op0);
2416 }
2417 return Register();
2418}
2419
2420Register fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
2421 if (RetVT.SimpleTy != MVT::v8f16)
2422 return Register();
2423 if ((Subtarget->hasMVEFloatOps())) {
2424 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16M, RC: &ARM::MQPRRegClass, Op0);
2425 }
2426 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2427 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNQh, RC: &ARM::QPRRegClass, Op0);
2428 }
2429 return Register();
2430}
2431
2432Register fastEmit_ISD_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
2433 if (RetVT.SimpleTy != MVT::v2f32)
2434 return Register();
2435 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2436 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNDf, RC: &ARM::DPRRegClass, Op0);
2437 }
2438 return Register();
2439}
2440
2441Register fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
2442 if (RetVT.SimpleTy != MVT::v4f32)
2443 return Register();
2444 if ((Subtarget->hasMVEFloatOps())) {
2445 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32M, RC: &ARM::MQPRRegClass, Op0);
2446 }
2447 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2448 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNQf, RC: &ARM::QPRRegClass, Op0);
2449 }
2450 return Register();
2451}
2452
2453Register fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
2454 switch (VT.SimpleTy) {
2455 case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0);
2456 case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0);
2457 case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0);
2458 case MVT::v4f16: return fastEmit_ISD_FFLOOR_MVT_v4f16_r(RetVT, Op0);
2459 case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0);
2460 case MVT::v2f32: return fastEmit_ISD_FFLOOR_MVT_v2f32_r(RetVT, Op0);
2461 case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0);
2462 default: return Register();
2463 }
2464}
2465
2466// FastEmit functions for ISD::FNEARBYINT.
2467
2468Register fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
2469 if (RetVT.SimpleTy != MVT::f16)
2470 return Register();
2471 if ((Subtarget->hasFullFP16())) {
2472 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRH, RC: &ARM::HPRRegClass, Op0);
2473 }
2474 return Register();
2475}
2476
2477Register fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
2478 if (RetVT.SimpleTy != MVT::f32)
2479 return Register();
2480 if ((Subtarget->hasFPARMv8Base())) {
2481 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRS, RC: &ARM::SPRRegClass, Op0);
2482 }
2483 return Register();
2484}
2485
2486Register fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
2487 if (RetVT.SimpleTy != MVT::f64)
2488 return Register();
2489 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2490 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRD, RC: &ARM::DPRRegClass, Op0);
2491 }
2492 return Register();
2493}
2494
2495Register fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
2496 switch (VT.SimpleTy) {
2497 case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0);
2498 case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0);
2499 case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0);
2500 default: return Register();
2501 }
2502}
2503
2504// FastEmit functions for ISD::FNEG.
2505
2506Register fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, Register Op0) {
2507 if (RetVT.SimpleTy != MVT::f16)
2508 return Register();
2509 if ((Subtarget->hasFullFP16())) {
2510 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGH, RC: &ARM::HPRRegClass, Op0);
2511 }
2512 return Register();
2513}
2514
2515Register fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, Register Op0) {
2516 if (RetVT.SimpleTy != MVT::f32)
2517 return Register();
2518 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
2519 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGS, RC: &ARM::SPRRegClass, Op0);
2520 }
2521 return Register();
2522}
2523
2524Register fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, Register Op0) {
2525 if (RetVT.SimpleTy != MVT::f64)
2526 return Register();
2527 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2528 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGD, RC: &ARM::DPRRegClass, Op0);
2529 }
2530 return Register();
2531}
2532
2533Register fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, Register Op0) {
2534 if (RetVT.SimpleTy != MVT::v4f16)
2535 return Register();
2536 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2537 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGhd, RC: &ARM::DPRRegClass, Op0);
2538 }
2539 return Register();
2540}
2541
2542Register fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, Register Op0) {
2543 if (RetVT.SimpleTy != MVT::v8f16)
2544 return Register();
2545 if ((Subtarget->hasMVEIntegerOps())) {
2546 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VNEGf16, RC: &ARM::MQPRRegClass, Op0);
2547 }
2548 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2549 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGhq, RC: &ARM::QPRRegClass, Op0);
2550 }
2551 return Register();
2552}
2553
2554Register fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, Register Op0) {
2555 if (RetVT.SimpleTy != MVT::v2f32)
2556 return Register();
2557 if ((Subtarget->hasNEON())) {
2558 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGfd, RC: &ARM::DPRRegClass, Op0);
2559 }
2560 return Register();
2561}
2562
2563Register fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, Register Op0) {
2564 if (RetVT.SimpleTy != MVT::v4f32)
2565 return Register();
2566 if ((Subtarget->hasMVEIntegerOps())) {
2567 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VNEGf32, RC: &ARM::MQPRRegClass, Op0);
2568 }
2569 if ((Subtarget->hasNEON())) {
2570 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGf32q, RC: &ARM::QPRRegClass, Op0);
2571 }
2572 return Register();
2573}
2574
2575Register fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, Register Op0) {
2576 switch (VT.SimpleTy) {
2577 case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0);
2578 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
2579 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
2580 case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0);
2581 case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0);
2582 case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0);
2583 case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0);
2584 default: return Register();
2585 }
2586}
2587
2588// FastEmit functions for ISD::FP_EXTEND.
2589
2590Register fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
2591 if (RetVT.SimpleTy != MVT::f64)
2592 return Register();
2593 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2594 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTDS, RC: &ARM::DPRRegClass, Op0);
2595 }
2596 return Register();
2597}
2598
2599Register fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
2600 if (RetVT.SimpleTy != MVT::v4f32)
2601 return Register();
2602 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2f, RC: &ARM::QPRRegClass, Op0);
2603}
2604
2605Register fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
2606 switch (VT.SimpleTy) {
2607 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
2608 case MVT::v4f16: return fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
2609 default: return Register();
2610 }
2611}
2612
2613// FastEmit functions for ISD::FP_ROUND.
2614
2615Register fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
2616 if (RetVT.SimpleTy != MVT::f32)
2617 return Register();
2618 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2619 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTSD, RC: &ARM::SPRRegClass, Op0);
2620 }
2621 return Register();
2622}
2623
2624Register fastEmit_ISD_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
2625 if (RetVT.SimpleTy != MVT::v4f16)
2626 return Register();
2627 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2h, RC: &ARM::DPRRegClass, Op0);
2628}
2629
2630Register fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
2631 switch (VT.SimpleTy) {
2632 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
2633 case MVT::v4f32: return fastEmit_ISD_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
2634 default: return Register();
2635 }
2636}
2637
2638// FastEmit functions for ISD::FP_TO_SINT.
2639
2640Register fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2641 if (RetVT.SimpleTy != MVT::v4i16)
2642 return Register();
2643 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2644 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2sd, RC: &ARM::DPRRegClass, Op0);
2645 }
2646 return Register();
2647}
2648
2649Register fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2650 if (RetVT.SimpleTy != MVT::v8i16)
2651 return Register();
2652 if ((Subtarget->hasMVEFloatOps())) {
2653 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTs16f16z, RC: &ARM::MQPRRegClass, Op0);
2654 }
2655 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2656 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2sq, RC: &ARM::QPRRegClass, Op0);
2657 }
2658 return Register();
2659}
2660
2661Register fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2662 if (RetVT.SimpleTy != MVT::v2i32)
2663 return Register();
2664 if ((Subtarget->hasNEON())) {
2665 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2sd, RC: &ARM::DPRRegClass, Op0);
2666 }
2667 return Register();
2668}
2669
2670Register fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2671 if (RetVT.SimpleTy != MVT::v4i32)
2672 return Register();
2673 if ((Subtarget->hasMVEFloatOps())) {
2674 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTs32f32z, RC: &ARM::MQPRRegClass, Op0);
2675 }
2676 if ((Subtarget->hasNEON())) {
2677 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2sq, RC: &ARM::QPRRegClass, Op0);
2678 }
2679 return Register();
2680}
2681
2682Register fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
2683 switch (VT.SimpleTy) {
2684 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
2685 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
2686 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
2687 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
2688 default: return Register();
2689 }
2690}
2691
2692// FastEmit functions for ISD::FP_TO_UINT.
2693
2694Register fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2695 if (RetVT.SimpleTy != MVT::v4i16)
2696 return Register();
2697 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2698 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2ud, RC: &ARM::DPRRegClass, Op0);
2699 }
2700 return Register();
2701}
2702
2703Register fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2704 if (RetVT.SimpleTy != MVT::v8i16)
2705 return Register();
2706 if ((Subtarget->hasMVEFloatOps())) {
2707 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTu16f16z, RC: &ARM::MQPRRegClass, Op0);
2708 }
2709 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2710 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2uq, RC: &ARM::QPRRegClass, Op0);
2711 }
2712 return Register();
2713}
2714
2715Register fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2716 if (RetVT.SimpleTy != MVT::v2i32)
2717 return Register();
2718 if ((Subtarget->hasNEON())) {
2719 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2ud, RC: &ARM::DPRRegClass, Op0);
2720 }
2721 return Register();
2722}
2723
2724Register fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2725 if (RetVT.SimpleTy != MVT::v4i32)
2726 return Register();
2727 if ((Subtarget->hasMVEFloatOps())) {
2728 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTu32f32z, RC: &ARM::MQPRRegClass, Op0);
2729 }
2730 if ((Subtarget->hasNEON())) {
2731 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2uq, RC: &ARM::QPRRegClass, Op0);
2732 }
2733 return Register();
2734}
2735
2736Register fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
2737 switch (VT.SimpleTy) {
2738 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
2739 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
2740 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
2741 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
2742 default: return Register();
2743 }
2744}
2745
2746// FastEmit functions for ISD::FRINT.
2747
2748Register fastEmit_ISD_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
2749 if (RetVT.SimpleTy != MVT::f16)
2750 return Register();
2751 if ((Subtarget->hasFullFP16())) {
2752 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXH, RC: &ARM::HPRRegClass, Op0);
2753 }
2754 return Register();
2755}
2756
2757Register fastEmit_ISD_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
2758 if (RetVT.SimpleTy != MVT::f32)
2759 return Register();
2760 if ((Subtarget->hasFPARMv8Base())) {
2761 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXS, RC: &ARM::SPRRegClass, Op0);
2762 }
2763 return Register();
2764}
2765
2766Register fastEmit_ISD_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
2767 if (RetVT.SimpleTy != MVT::f64)
2768 return Register();
2769 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2770 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXD, RC: &ARM::DPRRegClass, Op0);
2771 }
2772 return Register();
2773}
2774
2775Register fastEmit_ISD_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2776 if (RetVT.SimpleTy != MVT::v4f16)
2777 return Register();
2778 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2779 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNDh, RC: &ARM::DPRRegClass, Op0);
2780 }
2781 return Register();
2782}
2783
2784Register fastEmit_ISD_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2785 if (RetVT.SimpleTy != MVT::v8f16)
2786 return Register();
2787 if ((Subtarget->hasMVEFloatOps())) {
2788 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16X, RC: &ARM::MQPRRegClass, Op0);
2789 }
2790 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2791 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNQh, RC: &ARM::QPRRegClass, Op0);
2792 }
2793 return Register();
2794}
2795
2796Register fastEmit_ISD_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2797 if (RetVT.SimpleTy != MVT::v2f32)
2798 return Register();
2799 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2800 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNDf, RC: &ARM::DPRRegClass, Op0);
2801 }
2802 return Register();
2803}
2804
2805Register fastEmit_ISD_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2806 if (RetVT.SimpleTy != MVT::v4f32)
2807 return Register();
2808 if ((Subtarget->hasMVEFloatOps())) {
2809 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32X, RC: &ARM::MQPRRegClass, Op0);
2810 }
2811 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2812 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNQf, RC: &ARM::QPRRegClass, Op0);
2813 }
2814 return Register();
2815}
2816
2817Register fastEmit_ISD_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
2818 switch (VT.SimpleTy) {
2819 case MVT::f16: return fastEmit_ISD_FRINT_MVT_f16_r(RetVT, Op0);
2820 case MVT::f32: return fastEmit_ISD_FRINT_MVT_f32_r(RetVT, Op0);
2821 case MVT::f64: return fastEmit_ISD_FRINT_MVT_f64_r(RetVT, Op0);
2822 case MVT::v4f16: return fastEmit_ISD_FRINT_MVT_v4f16_r(RetVT, Op0);
2823 case MVT::v8f16: return fastEmit_ISD_FRINT_MVT_v8f16_r(RetVT, Op0);
2824 case MVT::v2f32: return fastEmit_ISD_FRINT_MVT_v2f32_r(RetVT, Op0);
2825 case MVT::v4f32: return fastEmit_ISD_FRINT_MVT_v4f32_r(RetVT, Op0);
2826 default: return Register();
2827 }
2828}
2829
2830// FastEmit functions for ISD::FROUND.
2831
2832Register fastEmit_ISD_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
2833 if (RetVT.SimpleTy != MVT::f16)
2834 return Register();
2835 if ((Subtarget->hasFullFP16())) {
2836 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAH, RC: &ARM::HPRRegClass, Op0);
2837 }
2838 return Register();
2839}
2840
2841Register fastEmit_ISD_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
2842 if (RetVT.SimpleTy != MVT::f32)
2843 return Register();
2844 if ((Subtarget->hasFPARMv8Base())) {
2845 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAS, RC: &ARM::SPRRegClass, Op0);
2846 }
2847 return Register();
2848}
2849
2850Register fastEmit_ISD_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
2851 if (RetVT.SimpleTy != MVT::f64)
2852 return Register();
2853 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2854 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAD, RC: &ARM::DPRRegClass, Op0);
2855 }
2856 return Register();
2857}
2858
2859Register fastEmit_ISD_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
2860 if (RetVT.SimpleTy != MVT::v4f16)
2861 return Register();
2862 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2863 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANDh, RC: &ARM::DPRRegClass, Op0);
2864 }
2865 return Register();
2866}
2867
2868Register fastEmit_ISD_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
2869 if (RetVT.SimpleTy != MVT::v8f16)
2870 return Register();
2871 if ((Subtarget->hasMVEFloatOps())) {
2872 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16A, RC: &ARM::MQPRRegClass, Op0);
2873 }
2874 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2875 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANQh, RC: &ARM::QPRRegClass, Op0);
2876 }
2877 return Register();
2878}
2879
2880Register fastEmit_ISD_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
2881 if (RetVT.SimpleTy != MVT::v2f32)
2882 return Register();
2883 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2884 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANDf, RC: &ARM::DPRRegClass, Op0);
2885 }
2886 return Register();
2887}
2888
2889Register fastEmit_ISD_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
2890 if (RetVT.SimpleTy != MVT::v4f32)
2891 return Register();
2892 if ((Subtarget->hasMVEFloatOps())) {
2893 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32A, RC: &ARM::MQPRRegClass, Op0);
2894 }
2895 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2896 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANQf, RC: &ARM::QPRRegClass, Op0);
2897 }
2898 return Register();
2899}
2900
2901Register fastEmit_ISD_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
2902 switch (VT.SimpleTy) {
2903 case MVT::f16: return fastEmit_ISD_FROUND_MVT_f16_r(RetVT, Op0);
2904 case MVT::f32: return fastEmit_ISD_FROUND_MVT_f32_r(RetVT, Op0);
2905 case MVT::f64: return fastEmit_ISD_FROUND_MVT_f64_r(RetVT, Op0);
2906 case MVT::v4f16: return fastEmit_ISD_FROUND_MVT_v4f16_r(RetVT, Op0);
2907 case MVT::v8f16: return fastEmit_ISD_FROUND_MVT_v8f16_r(RetVT, Op0);
2908 case MVT::v2f32: return fastEmit_ISD_FROUND_MVT_v2f32_r(RetVT, Op0);
2909 case MVT::v4f32: return fastEmit_ISD_FROUND_MVT_v4f32_r(RetVT, Op0);
2910 default: return Register();
2911 }
2912}
2913
2914// FastEmit functions for ISD::FROUNDEVEN.
2915
2916Register fastEmit_ISD_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
2917 if (RetVT.SimpleTy != MVT::f16)
2918 return Register();
2919 if ((Subtarget->hasFullFP16())) {
2920 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNH, RC: &ARM::HPRRegClass, Op0);
2921 }
2922 return Register();
2923}
2924
2925Register fastEmit_ISD_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
2926 if (RetVT.SimpleTy != MVT::f32)
2927 return Register();
2928 if ((Subtarget->hasFPARMv8Base())) {
2929 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNS, RC: &ARM::SPRRegClass, Op0);
2930 }
2931 return Register();
2932}
2933
2934Register fastEmit_ISD_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
2935 if (RetVT.SimpleTy != MVT::f64)
2936 return Register();
2937 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2938 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTND, RC: &ARM::DPRRegClass, Op0);
2939 }
2940 return Register();
2941}
2942
2943Register fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
2944 if (RetVT.SimpleTy != MVT::v4f16)
2945 return Register();
2946 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2947 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNDh, RC: &ARM::DPRRegClass, Op0);
2948 }
2949 return Register();
2950}
2951
2952Register fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
2953 if (RetVT.SimpleTy != MVT::v8f16)
2954 return Register();
2955 if ((Subtarget->hasMVEFloatOps())) {
2956 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16N, RC: &ARM::MQPRRegClass, Op0);
2957 }
2958 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2959 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNQh, RC: &ARM::QPRRegClass, Op0);
2960 }
2961 return Register();
2962}
2963
2964Register fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
2965 if (RetVT.SimpleTy != MVT::v2f32)
2966 return Register();
2967 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2968 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNDf, RC: &ARM::DPRRegClass, Op0);
2969 }
2970 return Register();
2971}
2972
2973Register fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
2974 if (RetVT.SimpleTy != MVT::v4f32)
2975 return Register();
2976 if ((Subtarget->hasMVEFloatOps())) {
2977 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32N, RC: &ARM::MQPRRegClass, Op0);
2978 }
2979 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2980 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNQf, RC: &ARM::QPRRegClass, Op0);
2981 }
2982 return Register();
2983}
2984
2985Register fastEmit_ISD_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
2986 switch (VT.SimpleTy) {
2987 case MVT::f16: return fastEmit_ISD_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
2988 case MVT::f32: return fastEmit_ISD_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
2989 case MVT::f64: return fastEmit_ISD_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
2990 case MVT::v4f16: return fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
2991 case MVT::v8f16: return fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
2992 case MVT::v2f32: return fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
2993 case MVT::v4f32: return fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
2994 default: return Register();
2995 }
2996}
2997
2998// FastEmit functions for ISD::FSQRT.
2999
3000Register fastEmit_ISD_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
3001 if (RetVT.SimpleTy != MVT::f16)
3002 return Register();
3003 if ((Subtarget->hasFullFP16())) {
3004 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTH, RC: &ARM::HPRRegClass, Op0);
3005 }
3006 return Register();
3007}
3008
3009Register fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
3010 if (RetVT.SimpleTy != MVT::f32)
3011 return Register();
3012 if ((Subtarget->hasVFP2Base())) {
3013 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTS, RC: &ARM::SPRRegClass, Op0);
3014 }
3015 return Register();
3016}
3017
3018Register fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
3019 if (RetVT.SimpleTy != MVT::f64)
3020 return Register();
3021 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3022 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTD, RC: &ARM::DPRRegClass, Op0);
3023 }
3024 return Register();
3025}
3026
3027Register fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
3028 switch (VT.SimpleTy) {
3029 case MVT::f16: return fastEmit_ISD_FSQRT_MVT_f16_r(RetVT, Op0);
3030 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
3031 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
3032 default: return Register();
3033 }
3034}
3035
3036// FastEmit functions for ISD::FTRUNC.
3037
3038Register fastEmit_ISD_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
3039 if (RetVT.SimpleTy != MVT::f16)
3040 return Register();
3041 if ((Subtarget->hasFullFP16())) {
3042 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZH, RC: &ARM::HPRRegClass, Op0);
3043 }
3044 return Register();
3045}
3046
3047Register fastEmit_ISD_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
3048 if (RetVT.SimpleTy != MVT::f32)
3049 return Register();
3050 if ((Subtarget->hasFPARMv8Base())) {
3051 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZS, RC: &ARM::SPRRegClass, Op0);
3052 }
3053 return Register();
3054}
3055
3056Register fastEmit_ISD_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
3057 if (RetVT.SimpleTy != MVT::f64)
3058 return Register();
3059 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3060 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZD, RC: &ARM::DPRRegClass, Op0);
3061 }
3062 return Register();
3063}
3064
3065Register fastEmit_ISD_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
3066 if (RetVT.SimpleTy != MVT::v4f16)
3067 return Register();
3068 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3069 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNDh, RC: &ARM::DPRRegClass, Op0);
3070 }
3071 return Register();
3072}
3073
3074Register fastEmit_ISD_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
3075 if (RetVT.SimpleTy != MVT::v8f16)
3076 return Register();
3077 if ((Subtarget->hasMVEFloatOps())) {
3078 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16Z, RC: &ARM::MQPRRegClass, Op0);
3079 }
3080 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3081 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNQh, RC: &ARM::QPRRegClass, Op0);
3082 }
3083 return Register();
3084}
3085
3086Register fastEmit_ISD_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
3087 if (RetVT.SimpleTy != MVT::v2f32)
3088 return Register();
3089 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3090 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNDf, RC: &ARM::DPRRegClass, Op0);
3091 }
3092 return Register();
3093}
3094
3095Register fastEmit_ISD_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
3096 if (RetVT.SimpleTy != MVT::v4f32)
3097 return Register();
3098 if ((Subtarget->hasMVEFloatOps())) {
3099 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32Z, RC: &ARM::MQPRRegClass, Op0);
3100 }
3101 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3102 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNQf, RC: &ARM::QPRRegClass, Op0);
3103 }
3104 return Register();
3105}
3106
3107Register fastEmit_ISD_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
3108 switch (VT.SimpleTy) {
3109 case MVT::f16: return fastEmit_ISD_FTRUNC_MVT_f16_r(RetVT, Op0);
3110 case MVT::f32: return fastEmit_ISD_FTRUNC_MVT_f32_r(RetVT, Op0);
3111 case MVT::f64: return fastEmit_ISD_FTRUNC_MVT_f64_r(RetVT, Op0);
3112 case MVT::v4f16: return fastEmit_ISD_FTRUNC_MVT_v4f16_r(RetVT, Op0);
3113 case MVT::v8f16: return fastEmit_ISD_FTRUNC_MVT_v8f16_r(RetVT, Op0);
3114 case MVT::v2f32: return fastEmit_ISD_FTRUNC_MVT_v2f32_r(RetVT, Op0);
3115 case MVT::v4f32: return fastEmit_ISD_FTRUNC_MVT_v4f32_r(RetVT, Op0);
3116 default: return Register();
3117 }
3118}
3119
3120// FastEmit functions for ISD::SET_FPENV.
3121
3122Register fastEmit_ISD_SET_FPENV_MVT_i32_r(MVT RetVT, Register Op0) {
3123 if (RetVT.SimpleTy != MVT::isVoid)
3124 return Register();
3125 return fastEmitInst_r(MachineInstOpcode: ARM::VMSR, RC: &ARM::GPRnopcRegClass, Op0);
3126}
3127
3128Register fastEmit_ISD_SET_FPENV_r(MVT VT, MVT RetVT, Register Op0) {
3129 switch (VT.SimpleTy) {
3130 case MVT::i32: return fastEmit_ISD_SET_FPENV_MVT_i32_r(RetVT, Op0);
3131 default: return Register();
3132 }
3133}
3134
3135// FastEmit functions for ISD::SIGN_EXTEND.
3136
3137Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i8_r(MVT RetVT, Register Op0) {
3138 if (RetVT.SimpleTy != MVT::v8i16)
3139 return Register();
3140 if ((Subtarget->hasNEON())) {
3141 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLsv8i16, RC: &ARM::QPRRegClass, Op0);
3142 }
3143 return Register();
3144}
3145
3146Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i16_r(MVT RetVT, Register Op0) {
3147 if (RetVT.SimpleTy != MVT::v4i32)
3148 return Register();
3149 if ((Subtarget->hasNEON())) {
3150 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLsv4i32, RC: &ARM::QPRRegClass, Op0);
3151 }
3152 return Register();
3153}
3154
3155Register fastEmit_ISD_SIGN_EXTEND_MVT_v2i32_r(MVT RetVT, Register Op0) {
3156 if (RetVT.SimpleTy != MVT::v2i64)
3157 return Register();
3158 if ((Subtarget->hasNEON())) {
3159 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLsv2i64, RC: &ARM::QPRRegClass, Op0);
3160 }
3161 return Register();
3162}
3163
3164Register fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3165 switch (VT.SimpleTy) {
3166 case MVT::v8i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i8_r(RetVT, Op0);
3167 case MVT::v4i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i16_r(RetVT, Op0);
3168 case MVT::v2i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i32_r(RetVT, Op0);
3169 default: return Register();
3170 }
3171}
3172
3173// FastEmit functions for ISD::SINT_TO_FP.
3174
3175Register fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
3176 if (RetVT.SimpleTy != MVT::v4f16)
3177 return Register();
3178 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3179 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2hd, RC: &ARM::DPRRegClass, Op0);
3180 }
3181 return Register();
3182}
3183
3184Register fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
3185 if (RetVT.SimpleTy != MVT::v8f16)
3186 return Register();
3187 if ((Subtarget->hasMVEFloatOps())) {
3188 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf16s16n, RC: &ARM::MQPRRegClass, Op0);
3189 }
3190 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3191 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2hq, RC: &ARM::QPRRegClass, Op0);
3192 }
3193 return Register();
3194}
3195
3196Register fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
3197 if (RetVT.SimpleTy != MVT::v2f32)
3198 return Register();
3199 if ((Subtarget->hasNEON())) {
3200 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2fd, RC: &ARM::DPRRegClass, Op0);
3201 }
3202 return Register();
3203}
3204
3205Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
3206 if (RetVT.SimpleTy != MVT::v4f32)
3207 return Register();
3208 if ((Subtarget->hasMVEFloatOps())) {
3209 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf32s32n, RC: &ARM::MQPRRegClass, Op0);
3210 }
3211 if ((Subtarget->hasNEON())) {
3212 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2fq, RC: &ARM::QPRRegClass, Op0);
3213 }
3214 return Register();
3215}
3216
3217Register fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3218 switch (VT.SimpleTy) {
3219 case MVT::v4i16: return fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
3220 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3221 case MVT::v2i32: return fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
3222 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3223 default: return Register();
3224 }
3225}
3226
3227// FastEmit functions for ISD::STRICT_FCEIL.
3228
3229Register fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
3230 if (RetVT.SimpleTy != MVT::f16)
3231 return Register();
3232 if ((Subtarget->hasFullFP16())) {
3233 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPH, RC: &ARM::HPRRegClass, Op0);
3234 }
3235 return Register();
3236}
3237
3238Register fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
3239 if (RetVT.SimpleTy != MVT::f32)
3240 return Register();
3241 if ((Subtarget->hasFPARMv8Base())) {
3242 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPS, RC: &ARM::SPRRegClass, Op0);
3243 }
3244 return Register();
3245}
3246
3247Register fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
3248 if (RetVT.SimpleTy != MVT::f64)
3249 return Register();
3250 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3251 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPD, RC: &ARM::DPRRegClass, Op0);
3252 }
3253 return Register();
3254}
3255
3256Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
3257 if (RetVT.SimpleTy != MVT::v4f16)
3258 return Register();
3259 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3260 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNDh, RC: &ARM::DPRRegClass, Op0);
3261 }
3262 return Register();
3263}
3264
3265Register fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
3266 if (RetVT.SimpleTy != MVT::v8f16)
3267 return Register();
3268 if ((Subtarget->hasMVEFloatOps())) {
3269 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16P, RC: &ARM::MQPRRegClass, Op0);
3270 }
3271 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3272 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNQh, RC: &ARM::QPRRegClass, Op0);
3273 }
3274 return Register();
3275}
3276
3277Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
3278 if (RetVT.SimpleTy != MVT::v2f32)
3279 return Register();
3280 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3281 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNDf, RC: &ARM::DPRRegClass, Op0);
3282 }
3283 return Register();
3284}
3285
3286Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
3287 if (RetVT.SimpleTy != MVT::v4f32)
3288 return Register();
3289 if ((Subtarget->hasMVEFloatOps())) {
3290 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32P, RC: &ARM::MQPRRegClass, Op0);
3291 }
3292 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3293 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNQf, RC: &ARM::QPRRegClass, Op0);
3294 }
3295 return Register();
3296}
3297
3298Register fastEmit_ISD_STRICT_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
3299 switch (VT.SimpleTy) {
3300 case MVT::f16: return fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(RetVT, Op0);
3301 case MVT::f32: return fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(RetVT, Op0);
3302 case MVT::f64: return fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(RetVT, Op0);
3303 case MVT::v4f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(RetVT, Op0);
3304 case MVT::v8f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(RetVT, Op0);
3305 case MVT::v2f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(RetVT, Op0);
3306 case MVT::v4f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(RetVT, Op0);
3307 default: return Register();
3308 }
3309}
3310
3311// FastEmit functions for ISD::STRICT_FFLOOR.
3312
3313Register fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
3314 if (RetVT.SimpleTy != MVT::f16)
3315 return Register();
3316 if ((Subtarget->hasFullFP16())) {
3317 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMH, RC: &ARM::HPRRegClass, Op0);
3318 }
3319 return Register();
3320}
3321
3322Register fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
3323 if (RetVT.SimpleTy != MVT::f32)
3324 return Register();
3325 if ((Subtarget->hasFPARMv8Base())) {
3326 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMS, RC: &ARM::SPRRegClass, Op0);
3327 }
3328 return Register();
3329}
3330
3331Register fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
3332 if (RetVT.SimpleTy != MVT::f64)
3333 return Register();
3334 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3335 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMD, RC: &ARM::DPRRegClass, Op0);
3336 }
3337 return Register();
3338}
3339
3340Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
3341 if (RetVT.SimpleTy != MVT::v4f16)
3342 return Register();
3343 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3344 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNDh, RC: &ARM::DPRRegClass, Op0);
3345 }
3346 return Register();
3347}
3348
3349Register fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
3350 if (RetVT.SimpleTy != MVT::v8f16)
3351 return Register();
3352 if ((Subtarget->hasMVEFloatOps())) {
3353 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16M, RC: &ARM::MQPRRegClass, Op0);
3354 }
3355 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3356 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNQh, RC: &ARM::QPRRegClass, Op0);
3357 }
3358 return Register();
3359}
3360
3361Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
3362 if (RetVT.SimpleTy != MVT::v2f32)
3363 return Register();
3364 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3365 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNDf, RC: &ARM::DPRRegClass, Op0);
3366 }
3367 return Register();
3368}
3369
3370Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
3371 if (RetVT.SimpleTy != MVT::v4f32)
3372 return Register();
3373 if ((Subtarget->hasMVEFloatOps())) {
3374 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32M, RC: &ARM::MQPRRegClass, Op0);
3375 }
3376 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3377 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNQf, RC: &ARM::QPRRegClass, Op0);
3378 }
3379 return Register();
3380}
3381
3382Register fastEmit_ISD_STRICT_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
3383 switch (VT.SimpleTy) {
3384 case MVT::f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(RetVT, Op0);
3385 case MVT::f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(RetVT, Op0);
3386 case MVT::f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(RetVT, Op0);
3387 case MVT::v4f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(RetVT, Op0);
3388 case MVT::v8f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(RetVT, Op0);
3389 case MVT::v2f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(RetVT, Op0);
3390 case MVT::v4f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(RetVT, Op0);
3391 default: return Register();
3392 }
3393}
3394
3395// FastEmit functions for ISD::STRICT_FNEARBYINT.
3396
3397Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
3398 if (RetVT.SimpleTy != MVT::f16)
3399 return Register();
3400 if ((Subtarget->hasFullFP16())) {
3401 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRH, RC: &ARM::HPRRegClass, Op0);
3402 }
3403 return Register();
3404}
3405
3406Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
3407 if (RetVT.SimpleTy != MVT::f32)
3408 return Register();
3409 if ((Subtarget->hasFPARMv8Base())) {
3410 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRS, RC: &ARM::SPRRegClass, Op0);
3411 }
3412 return Register();
3413}
3414
3415Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
3416 if (RetVT.SimpleTy != MVT::f64)
3417 return Register();
3418 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3419 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRD, RC: &ARM::DPRRegClass, Op0);
3420 }
3421 return Register();
3422}
3423
3424Register fastEmit_ISD_STRICT_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
3425 switch (VT.SimpleTy) {
3426 case MVT::f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(RetVT, Op0);
3427 case MVT::f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(RetVT, Op0);
3428 case MVT::f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(RetVT, Op0);
3429 default: return Register();
3430 }
3431}
3432
3433// FastEmit functions for ISD::STRICT_FP_EXTEND.
3434
3435Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
3436 if (RetVT.SimpleTy != MVT::f64)
3437 return Register();
3438 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3439 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTDS, RC: &ARM::DPRRegClass, Op0);
3440 }
3441 return Register();
3442}
3443
3444Register fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3445 switch (VT.SimpleTy) {
3446 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0);
3447 default: return Register();
3448 }
3449}
3450
3451// FastEmit functions for ISD::STRICT_FP_ROUND.
3452
3453Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3454 if (RetVT.SimpleTy != MVT::f32)
3455 return Register();
3456 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3457 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTSD, RC: &ARM::SPRRegClass, Op0);
3458 }
3459 return Register();
3460}
3461
3462Register fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
3463 switch (VT.SimpleTy) {
3464 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0);
3465 default: return Register();
3466 }
3467}
3468
3469// FastEmit functions for ISD::STRICT_FRINT.
3470
3471Register fastEmit_ISD_STRICT_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
3472 if (RetVT.SimpleTy != MVT::f16)
3473 return Register();
3474 if ((Subtarget->hasFullFP16())) {
3475 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXH, RC: &ARM::HPRRegClass, Op0);
3476 }
3477 return Register();
3478}
3479
3480Register fastEmit_ISD_STRICT_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
3481 if (RetVT.SimpleTy != MVT::f32)
3482 return Register();
3483 if ((Subtarget->hasFPARMv8Base())) {
3484 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXS, RC: &ARM::SPRRegClass, Op0);
3485 }
3486 return Register();
3487}
3488
3489Register fastEmit_ISD_STRICT_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
3490 if (RetVT.SimpleTy != MVT::f64)
3491 return Register();
3492 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3493 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXD, RC: &ARM::DPRRegClass, Op0);
3494 }
3495 return Register();
3496}
3497
3498Register fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3499 if (RetVT.SimpleTy != MVT::v4f16)
3500 return Register();
3501 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3502 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNDh, RC: &ARM::DPRRegClass, Op0);
3503 }
3504 return Register();
3505}
3506
3507Register fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3508 if (RetVT.SimpleTy != MVT::v8f16)
3509 return Register();
3510 if ((Subtarget->hasMVEFloatOps())) {
3511 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16X, RC: &ARM::MQPRRegClass, Op0);
3512 }
3513 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3514 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNQh, RC: &ARM::QPRRegClass, Op0);
3515 }
3516 return Register();
3517}
3518
3519Register fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3520 if (RetVT.SimpleTy != MVT::v2f32)
3521 return Register();
3522 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3523 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNDf, RC: &ARM::DPRRegClass, Op0);
3524 }
3525 return Register();
3526}
3527
3528Register fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3529 if (RetVT.SimpleTy != MVT::v4f32)
3530 return Register();
3531 if ((Subtarget->hasMVEFloatOps())) {
3532 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32X, RC: &ARM::MQPRRegClass, Op0);
3533 }
3534 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3535 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNQf, RC: &ARM::QPRRegClass, Op0);
3536 }
3537 return Register();
3538}
3539
3540Register fastEmit_ISD_STRICT_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
3541 switch (VT.SimpleTy) {
3542 case MVT::f16: return fastEmit_ISD_STRICT_FRINT_MVT_f16_r(RetVT, Op0);
3543 case MVT::f32: return fastEmit_ISD_STRICT_FRINT_MVT_f32_r(RetVT, Op0);
3544 case MVT::f64: return fastEmit_ISD_STRICT_FRINT_MVT_f64_r(RetVT, Op0);
3545 case MVT::v4f16: return fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(RetVT, Op0);
3546 case MVT::v8f16: return fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(RetVT, Op0);
3547 case MVT::v2f32: return fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(RetVT, Op0);
3548 case MVT::v4f32: return fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(RetVT, Op0);
3549 default: return Register();
3550 }
3551}
3552
3553// FastEmit functions for ISD::STRICT_FROUND.
3554
3555Register fastEmit_ISD_STRICT_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
3556 if (RetVT.SimpleTy != MVT::f16)
3557 return Register();
3558 if ((Subtarget->hasFullFP16())) {
3559 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAH, RC: &ARM::HPRRegClass, Op0);
3560 }
3561 return Register();
3562}
3563
3564Register fastEmit_ISD_STRICT_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3565 if (RetVT.SimpleTy != MVT::f32)
3566 return Register();
3567 if ((Subtarget->hasFPARMv8Base())) {
3568 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAS, RC: &ARM::SPRRegClass, Op0);
3569 }
3570 return Register();
3571}
3572
3573Register fastEmit_ISD_STRICT_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3574 if (RetVT.SimpleTy != MVT::f64)
3575 return Register();
3576 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3577 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAD, RC: &ARM::DPRRegClass, Op0);
3578 }
3579 return Register();
3580}
3581
3582Register fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3583 if (RetVT.SimpleTy != MVT::v4f16)
3584 return Register();
3585 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3586 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANDh, RC: &ARM::DPRRegClass, Op0);
3587 }
3588 return Register();
3589}
3590
3591Register fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
3592 if (RetVT.SimpleTy != MVT::v8f16)
3593 return Register();
3594 if ((Subtarget->hasMVEFloatOps())) {
3595 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16A, RC: &ARM::MQPRRegClass, Op0);
3596 }
3597 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3598 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANQh, RC: &ARM::QPRRegClass, Op0);
3599 }
3600 return Register();
3601}
3602
3603Register fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3604 if (RetVT.SimpleTy != MVT::v2f32)
3605 return Register();
3606 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3607 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANDf, RC: &ARM::DPRRegClass, Op0);
3608 }
3609 return Register();
3610}
3611
3612Register fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3613 if (RetVT.SimpleTy != MVT::v4f32)
3614 return Register();
3615 if ((Subtarget->hasMVEFloatOps())) {
3616 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32A, RC: &ARM::MQPRRegClass, Op0);
3617 }
3618 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3619 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANQf, RC: &ARM::QPRRegClass, Op0);
3620 }
3621 return Register();
3622}
3623
3624Register fastEmit_ISD_STRICT_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
3625 switch (VT.SimpleTy) {
3626 case MVT::f16: return fastEmit_ISD_STRICT_FROUND_MVT_f16_r(RetVT, Op0);
3627 case MVT::f32: return fastEmit_ISD_STRICT_FROUND_MVT_f32_r(RetVT, Op0);
3628 case MVT::f64: return fastEmit_ISD_STRICT_FROUND_MVT_f64_r(RetVT, Op0);
3629 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(RetVT, Op0);
3630 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(RetVT, Op0);
3631 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(RetVT, Op0);
3632 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(RetVT, Op0);
3633 default: return Register();
3634 }
3635}
3636
3637// FastEmit functions for ISD::STRICT_FROUNDEVEN.
3638
3639Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
3640 if (RetVT.SimpleTy != MVT::f16)
3641 return Register();
3642 if ((Subtarget->hasFullFP16())) {
3643 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNH, RC: &ARM::HPRRegClass, Op0);
3644 }
3645 return Register();
3646}
3647
3648Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
3649 if (RetVT.SimpleTy != MVT::f32)
3650 return Register();
3651 if ((Subtarget->hasFPARMv8Base())) {
3652 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNS, RC: &ARM::SPRRegClass, Op0);
3653 }
3654 return Register();
3655}
3656
3657Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
3658 if (RetVT.SimpleTy != MVT::f64)
3659 return Register();
3660 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3661 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTND, RC: &ARM::DPRRegClass, Op0);
3662 }
3663 return Register();
3664}
3665
3666Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
3667 if (RetVT.SimpleTy != MVT::v4f16)
3668 return Register();
3669 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3670 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNDh, RC: &ARM::DPRRegClass, Op0);
3671 }
3672 return Register();
3673}
3674
3675Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
3676 if (RetVT.SimpleTy != MVT::v8f16)
3677 return Register();
3678 if ((Subtarget->hasMVEFloatOps())) {
3679 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16N, RC: &ARM::MQPRRegClass, Op0);
3680 }
3681 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3682 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNQh, RC: &ARM::QPRRegClass, Op0);
3683 }
3684 return Register();
3685}
3686
3687Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
3688 if (RetVT.SimpleTy != MVT::v2f32)
3689 return Register();
3690 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3691 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNDf, RC: &ARM::DPRRegClass, Op0);
3692 }
3693 return Register();
3694}
3695
3696Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
3697 if (RetVT.SimpleTy != MVT::v4f32)
3698 return Register();
3699 if ((Subtarget->hasMVEFloatOps())) {
3700 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32N, RC: &ARM::MQPRRegClass, Op0);
3701 }
3702 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3703 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNQf, RC: &ARM::QPRRegClass, Op0);
3704 }
3705 return Register();
3706}
3707
3708Register fastEmit_ISD_STRICT_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
3709 switch (VT.SimpleTy) {
3710 case MVT::f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
3711 case MVT::f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
3712 case MVT::f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
3713 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
3714 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
3715 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
3716 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
3717 default: return Register();
3718 }
3719}
3720
3721// FastEmit functions for ISD::STRICT_FSQRT.
3722
3723Register fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
3724 if (RetVT.SimpleTy != MVT::f16)
3725 return Register();
3726 if ((Subtarget->hasFullFP16())) {
3727 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTH, RC: &ARM::HPRRegClass, Op0);
3728 }
3729 return Register();
3730}
3731
3732Register fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
3733 if (RetVT.SimpleTy != MVT::f32)
3734 return Register();
3735 if ((Subtarget->hasVFP2Base())) {
3736 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTS, RC: &ARM::SPRRegClass, Op0);
3737 }
3738 return Register();
3739}
3740
3741Register fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
3742 if (RetVT.SimpleTy != MVT::f64)
3743 return Register();
3744 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3745 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTD, RC: &ARM::DPRRegClass, Op0);
3746 }
3747 return Register();
3748}
3749
3750Register fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
3751 switch (VT.SimpleTy) {
3752 case MVT::f16: return fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(RetVT, Op0);
3753 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0);
3754 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0);
3755 default: return Register();
3756 }
3757}
3758
3759// FastEmit functions for ISD::STRICT_FTRUNC.
3760
3761Register fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
3762 if (RetVT.SimpleTy != MVT::f16)
3763 return Register();
3764 if ((Subtarget->hasFullFP16())) {
3765 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZH, RC: &ARM::HPRRegClass, Op0);
3766 }
3767 return Register();
3768}
3769
3770Register fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
3771 if (RetVT.SimpleTy != MVT::f32)
3772 return Register();
3773 if ((Subtarget->hasFPARMv8Base())) {
3774 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZS, RC: &ARM::SPRRegClass, Op0);
3775 }
3776 return Register();
3777}
3778
3779Register fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
3780 if (RetVT.SimpleTy != MVT::f64)
3781 return Register();
3782 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3783 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZD, RC: &ARM::DPRRegClass, Op0);
3784 }
3785 return Register();
3786}
3787
3788Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
3789 if (RetVT.SimpleTy != MVT::v4f16)
3790 return Register();
3791 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3792 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNDh, RC: &ARM::DPRRegClass, Op0);
3793 }
3794 return Register();
3795}
3796
3797Register fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
3798 if (RetVT.SimpleTy != MVT::v8f16)
3799 return Register();
3800 if ((Subtarget->hasMVEFloatOps())) {
3801 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16Z, RC: &ARM::MQPRRegClass, Op0);
3802 }
3803 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3804 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNQh, RC: &ARM::QPRRegClass, Op0);
3805 }
3806 return Register();
3807}
3808
3809Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
3810 if (RetVT.SimpleTy != MVT::v2f32)
3811 return Register();
3812 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3813 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNDf, RC: &ARM::DPRRegClass, Op0);
3814 }
3815 return Register();
3816}
3817
3818Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
3819 if (RetVT.SimpleTy != MVT::v4f32)
3820 return Register();
3821 if ((Subtarget->hasMVEFloatOps())) {
3822 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32Z, RC: &ARM::MQPRRegClass, Op0);
3823 }
3824 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3825 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNQf, RC: &ARM::QPRRegClass, Op0);
3826 }
3827 return Register();
3828}
3829
3830Register fastEmit_ISD_STRICT_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
3831 switch (VT.SimpleTy) {
3832 case MVT::f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(RetVT, Op0);
3833 case MVT::f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(RetVT, Op0);
3834 case MVT::f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(RetVT, Op0);
3835 case MVT::v4f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(RetVT, Op0);
3836 case MVT::v8f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(RetVT, Op0);
3837 case MVT::v2f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(RetVT, Op0);
3838 case MVT::v4f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(RetVT, Op0);
3839 default: return Register();
3840 }
3841}
3842
3843// FastEmit functions for ISD::TRUNCATE.
3844
3845Register fastEmit_ISD_TRUNCATE_MVT_v8i16_r(MVT RetVT, Register Op0) {
3846 if (RetVT.SimpleTy != MVT::v8i8)
3847 return Register();
3848 if ((Subtarget->hasNEON())) {
3849 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVNv8i8, RC: &ARM::DPRRegClass, Op0);
3850 }
3851 return Register();
3852}
3853
3854Register fastEmit_ISD_TRUNCATE_MVT_v4i32_r(MVT RetVT, Register Op0) {
3855 if (RetVT.SimpleTy != MVT::v4i16)
3856 return Register();
3857 if ((Subtarget->hasNEON())) {
3858 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVNv4i16, RC: &ARM::DPRRegClass, Op0);
3859 }
3860 return Register();
3861}
3862
3863Register fastEmit_ISD_TRUNCATE_MVT_v2i64_r(MVT RetVT, Register Op0) {
3864 if (RetVT.SimpleTy != MVT::v2i32)
3865 return Register();
3866 if ((Subtarget->hasNEON())) {
3867 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVNv2i32, RC: &ARM::DPRRegClass, Op0);
3868 }
3869 return Register();
3870}
3871
3872Register fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, Register Op0) {
3873 switch (VT.SimpleTy) {
3874 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i16_r(RetVT, Op0);
3875 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_MVT_v4i32_r(RetVT, Op0);
3876 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_MVT_v2i64_r(RetVT, Op0);
3877 default: return Register();
3878 }
3879}
3880
3881// FastEmit functions for ISD::UINT_TO_FP.
3882
3883Register fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
3884 if (RetVT.SimpleTy != MVT::v4f16)
3885 return Register();
3886 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3887 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2hd, RC: &ARM::DPRRegClass, Op0);
3888 }
3889 return Register();
3890}
3891
3892Register fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
3893 if (RetVT.SimpleTy != MVT::v8f16)
3894 return Register();
3895 if ((Subtarget->hasMVEFloatOps())) {
3896 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf16u16n, RC: &ARM::MQPRRegClass, Op0);
3897 }
3898 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3899 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2hq, RC: &ARM::QPRRegClass, Op0);
3900 }
3901 return Register();
3902}
3903
3904Register fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
3905 if (RetVT.SimpleTy != MVT::v2f32)
3906 return Register();
3907 if ((Subtarget->hasNEON())) {
3908 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2fd, RC: &ARM::DPRRegClass, Op0);
3909 }
3910 return Register();
3911}
3912
3913Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
3914 if (RetVT.SimpleTy != MVT::v4f32)
3915 return Register();
3916 if ((Subtarget->hasMVEFloatOps())) {
3917 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf32u32n, RC: &ARM::MQPRRegClass, Op0);
3918 }
3919 if ((Subtarget->hasNEON())) {
3920 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2fq, RC: &ARM::QPRRegClass, Op0);
3921 }
3922 return Register();
3923}
3924
3925Register fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3926 switch (VT.SimpleTy) {
3927 case MVT::v4i16: return fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
3928 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3929 case MVT::v2i32: return fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
3930 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3931 default: return Register();
3932 }
3933}
3934
3935// FastEmit functions for ISD::VECREDUCE_ADD.
3936
3937Register fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(MVT RetVT, Register Op0) {
3938 if (RetVT.SimpleTy != MVT::i32)
3939 return Register();
3940 if ((Subtarget->hasMVEIntegerOps())) {
3941 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu8no_acc, RC: &ARM::tGPREvenRegClass, Op0);
3942 }
3943 return Register();
3944}
3945
3946Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(MVT RetVT, Register Op0) {
3947 if (RetVT.SimpleTy != MVT::i32)
3948 return Register();
3949 if ((Subtarget->hasMVEIntegerOps())) {
3950 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu16no_acc, RC: &ARM::tGPREvenRegClass, Op0);
3951 }
3952 return Register();
3953}
3954
3955Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(MVT RetVT, Register Op0) {
3956 if (RetVT.SimpleTy != MVT::i32)
3957 return Register();
3958 if ((Subtarget->hasMVEIntegerOps())) {
3959 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu32no_acc, RC: &ARM::tGPREvenRegClass, Op0);
3960 }
3961 return Register();
3962}
3963
3964Register fastEmit_ISD_VECREDUCE_ADD_r(MVT VT, MVT RetVT, Register Op0) {
3965 switch (VT.SimpleTy) {
3966 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(RetVT, Op0);
3967 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(RetVT, Op0);
3968 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(RetVT, Op0);
3969 default: return Register();
3970 }
3971}
3972
3973// FastEmit functions for ISD::ZERO_EXTEND.
3974
3975Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i8_r(MVT RetVT, Register Op0) {
3976 if (RetVT.SimpleTy != MVT::v8i16)
3977 return Register();
3978 if ((Subtarget->hasNEON())) {
3979 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv8i16, RC: &ARM::QPRRegClass, Op0);
3980 }
3981 return Register();
3982}
3983
3984Register fastEmit_ISD_ZERO_EXTEND_MVT_v4i16_r(MVT RetVT, Register Op0) {
3985 if (RetVT.SimpleTy != MVT::v4i32)
3986 return Register();
3987 if ((Subtarget->hasNEON())) {
3988 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv4i32, RC: &ARM::QPRRegClass, Op0);
3989 }
3990 return Register();
3991}
3992
3993Register fastEmit_ISD_ZERO_EXTEND_MVT_v2i32_r(MVT RetVT, Register Op0) {
3994 if (RetVT.SimpleTy != MVT::v2i64)
3995 return Register();
3996 if ((Subtarget->hasNEON())) {
3997 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv2i64, RC: &ARM::QPRRegClass, Op0);
3998 }
3999 return Register();
4000}
4001
4002Register fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
4003 switch (VT.SimpleTy) {
4004 case MVT::v8i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i8_r(RetVT, Op0);
4005 case MVT::v4i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v4i16_r(RetVT, Op0);
4006 case MVT::v2i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v2i32_r(RetVT, Op0);
4007 default: return Register();
4008 }
4009}
4010
4011// Top-level FastEmit function.
4012
4013Register fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, Register Op0) override {
4014 switch (Opcode) {
4015 case ARMISD::CALL: return fastEmit_ARMISD_CALL_r(VT, RetVT, Op0);
4016 case ARMISD::CALL_NOLINK: return fastEmit_ARMISD_CALL_NOLINK_r(VT, RetVT, Op0);
4017 case ARMISD::CALL_PRED: return fastEmit_ARMISD_CALL_PRED_r(VT, RetVT, Op0);
4018 case ARMISD::CMPFPEw0: return fastEmit_ARMISD_CMPFPEw0_r(VT, RetVT, Op0);
4019 case ARMISD::CMPFPw0: return fastEmit_ARMISD_CMPFPw0_r(VT, RetVT, Op0);
4020 case ARMISD::VADDVs: return fastEmit_ARMISD_VADDVs_r(VT, RetVT, Op0);
4021 case ARMISD::VADDVu: return fastEmit_ARMISD_VADDVu_r(VT, RetVT, Op0);
4022 case ARMISD::VDUP: return fastEmit_ARMISD_VDUP_r(VT, RetVT, Op0);
4023 case ARMISD::VMOVSR: return fastEmit_ARMISD_VMOVSR_r(VT, RetVT, Op0);
4024 case ARMISD::VMOVhr: return fastEmit_ARMISD_VMOVhr_r(VT, RetVT, Op0);
4025 case ARMISD::VMOVrh: return fastEmit_ARMISD_VMOVrh_r(VT, RetVT, Op0);
4026 case ARMISD::VREV16: return fastEmit_ARMISD_VREV16_r(VT, RetVT, Op0);
4027 case ARMISD::VREV32: return fastEmit_ARMISD_VREV32_r(VT, RetVT, Op0);
4028 case ARMISD::VREV64: return fastEmit_ARMISD_VREV64_r(VT, RetVT, Op0);
4029 case ARMISD::WIN__DBZCHK: return fastEmit_ARMISD_WIN__DBZCHK_r(VT, RetVT, Op0);
4030 case ARMISD::WLSSETUP: return fastEmit_ARMISD_WLSSETUP_r(VT, RetVT, Op0);
4031 case ARMISD::tSECALL: return fastEmit_ARMISD_tSECALL_r(VT, RetVT, Op0);
4032 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
4033 case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0);
4034 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
4035 case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0);
4036 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0);
4037 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
4038 case ISD::CTLS: return fastEmit_ISD_CTLS_r(VT, RetVT, Op0);
4039 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
4040 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
4041 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
4042 case ISD::FCEIL: return fastEmit_ISD_FCEIL_r(VT, RetVT, Op0);
4043 case ISD::FFLOOR: return fastEmit_ISD_FFLOOR_r(VT, RetVT, Op0);
4044 case ISD::FNEARBYINT: return fastEmit_ISD_FNEARBYINT_r(VT, RetVT, Op0);
4045 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
4046 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
4047 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
4048 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
4049 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
4050 case ISD::FRINT: return fastEmit_ISD_FRINT_r(VT, RetVT, Op0);
4051 case ISD::FROUND: return fastEmit_ISD_FROUND_r(VT, RetVT, Op0);
4052 case ISD::FROUNDEVEN: return fastEmit_ISD_FROUNDEVEN_r(VT, RetVT, Op0);
4053 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
4054 case ISD::FTRUNC: return fastEmit_ISD_FTRUNC_r(VT, RetVT, Op0);
4055 case ISD::SET_FPENV: return fastEmit_ISD_SET_FPENV_r(VT, RetVT, Op0);
4056 case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0);
4057 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
4058 case ISD::STRICT_FCEIL: return fastEmit_ISD_STRICT_FCEIL_r(VT, RetVT, Op0);
4059 case ISD::STRICT_FFLOOR: return fastEmit_ISD_STRICT_FFLOOR_r(VT, RetVT, Op0);
4060 case ISD::STRICT_FNEARBYINT: return fastEmit_ISD_STRICT_FNEARBYINT_r(VT, RetVT, Op0);
4061 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0);
4062 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0);
4063 case ISD::STRICT_FRINT: return fastEmit_ISD_STRICT_FRINT_r(VT, RetVT, Op0);
4064 case ISD::STRICT_FROUND: return fastEmit_ISD_STRICT_FROUND_r(VT, RetVT, Op0);
4065 case ISD::STRICT_FROUNDEVEN: return fastEmit_ISD_STRICT_FROUNDEVEN_r(VT, RetVT, Op0);
4066 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0);
4067 case ISD::STRICT_FTRUNC: return fastEmit_ISD_STRICT_FTRUNC_r(VT, RetVT, Op0);
4068 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
4069 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
4070 case ISD::VECREDUCE_ADD: return fastEmit_ISD_VECREDUCE_ADD_r(VT, RetVT, Op0);
4071 case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0);
4072 default: return Register();
4073 }
4074}
4075
4076// FastEmit functions for ARMISD::CMN.
4077
4078Register fastEmit_ARMISD_CMN_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4079 if (RetVT.SimpleTy != MVT::i32)
4080 return Register();
4081 if ((Subtarget->isThumb2())) {
4082 return fastEmitInst_rr(MachineInstOpcode: ARM::t2CMNrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4083 }
4084 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
4085 return fastEmitInst_rr(MachineInstOpcode: ARM::tCMN, RC: &ARM::tGPRRegClass, Op0, Op1);
4086 }
4087 if ((!Subtarget->isThumb())) {
4088 return fastEmitInst_rr(MachineInstOpcode: ARM::CMNrr, RC: &ARM::GPRRegClass, Op0, Op1);
4089 }
4090 return Register();
4091}
4092
4093Register fastEmit_ARMISD_CMN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4094 switch (VT.SimpleTy) {
4095 case MVT::i32: return fastEmit_ARMISD_CMN_MVT_i32_rr(RetVT, Op0, Op1);
4096 default: return Register();
4097 }
4098}
4099
4100// FastEmit functions for ARMISD::CMP.
4101
4102Register fastEmit_ARMISD_CMP_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4103 if (RetVT.SimpleTy != MVT::i32)
4104 return Register();
4105 if ((Subtarget->isThumb2())) {
4106 return fastEmitInst_rr(MachineInstOpcode: ARM::t2CMPrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4107 }
4108 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
4109 return fastEmitInst_rr(MachineInstOpcode: ARM::tCMPr, RC: &ARM::tGPRRegClass, Op0, Op1);
4110 }
4111 if ((!Subtarget->isThumb())) {
4112 return fastEmitInst_rr(MachineInstOpcode: ARM::CMPrr, RC: &ARM::GPRRegClass, Op0, Op1);
4113 }
4114 return Register();
4115}
4116
4117Register fastEmit_ARMISD_CMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4118 switch (VT.SimpleTy) {
4119 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_rr(RetVT, Op0, Op1);
4120 default: return Register();
4121 }
4122}
4123
4124// FastEmit functions for ARMISD::CMPFP.
4125
4126Register fastEmit_ARMISD_CMPFP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
4127 if (RetVT.SimpleTy != MVT::i32)
4128 return Register();
4129 if ((Subtarget->hasFullFP16())) {
4130 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPH, RC: &ARM::HPRRegClass, Op0, Op1);
4131 }
4132 return Register();
4133}
4134
4135Register fastEmit_ARMISD_CMPFP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
4136 if (RetVT.SimpleTy != MVT::i32)
4137 return Register();
4138 if ((Subtarget->hasVFP2Base())) {
4139 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPS, RC: &ARM::SPRRegClass, Op0, Op1);
4140 }
4141 return Register();
4142}
4143
4144Register fastEmit_ARMISD_CMPFP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
4145 if (RetVT.SimpleTy != MVT::i32)
4146 return Register();
4147 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
4148 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPD, RC: &ARM::DPRRegClass, Op0, Op1);
4149 }
4150 return Register();
4151}
4152
4153Register fastEmit_ARMISD_CMPFP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4154 switch (VT.SimpleTy) {
4155 case MVT::f16: return fastEmit_ARMISD_CMPFP_MVT_f16_rr(RetVT, Op0, Op1);
4156 case MVT::f32: return fastEmit_ARMISD_CMPFP_MVT_f32_rr(RetVT, Op0, Op1);
4157 case MVT::f64: return fastEmit_ARMISD_CMPFP_MVT_f64_rr(RetVT, Op0, Op1);
4158 default: return Register();
4159 }
4160}
4161
4162// FastEmit functions for ARMISD::CMPFPE.
4163
4164Register fastEmit_ARMISD_CMPFPE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
4165 if (RetVT.SimpleTy != MVT::i32)
4166 return Register();
4167 if ((Subtarget->hasFullFP16())) {
4168 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPEH, RC: &ARM::HPRRegClass, Op0, Op1);
4169 }
4170 return Register();
4171}
4172
4173Register fastEmit_ARMISD_CMPFPE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
4174 if (RetVT.SimpleTy != MVT::i32)
4175 return Register();
4176 if ((Subtarget->hasVFP2Base())) {
4177 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPES, RC: &ARM::SPRRegClass, Op0, Op1);
4178 }
4179 return Register();
4180}
4181
4182Register fastEmit_ARMISD_CMPFPE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
4183 if (RetVT.SimpleTy != MVT::i32)
4184 return Register();
4185 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
4186 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPED, RC: &ARM::DPRRegClass, Op0, Op1);
4187 }
4188 return Register();
4189}
4190
4191Register fastEmit_ARMISD_CMPFPE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4192 switch (VT.SimpleTy) {
4193 case MVT::f16: return fastEmit_ARMISD_CMPFPE_MVT_f16_rr(RetVT, Op0, Op1);
4194 case MVT::f32: return fastEmit_ARMISD_CMPFPE_MVT_f32_rr(RetVT, Op0, Op1);
4195 case MVT::f64: return fastEmit_ARMISD_CMPFPE_MVT_f64_rr(RetVT, Op0, Op1);
4196 default: return Register();
4197 }
4198}
4199
4200// FastEmit functions for ARMISD::CMPZ.
4201
4202Register fastEmit_ARMISD_CMPZ_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4203 if (RetVT.SimpleTy != MVT::i32)
4204 return Register();
4205 if ((Subtarget->isThumb2())) {
4206 return fastEmitInst_rr(MachineInstOpcode: ARM::t2CMPrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4207 }
4208 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
4209 return fastEmitInst_rr(MachineInstOpcode: ARM::tCMPr, RC: &ARM::tGPRRegClass, Op0, Op1);
4210 }
4211 if ((!Subtarget->isThumb())) {
4212 return fastEmitInst_rr(MachineInstOpcode: ARM::CMPrr, RC: &ARM::GPRRegClass, Op0, Op1);
4213 }
4214 return Register();
4215}
4216
4217Register fastEmit_ARMISD_CMPZ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4218 switch (VT.SimpleTy) {
4219 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_rr(RetVT, Op0, Op1);
4220 default: return Register();
4221 }
4222}
4223
4224// FastEmit functions for ARMISD::EH_SJLJ_LONGJMP.
4225
4226Register fastEmit_ARMISD_EH_SJLJ_LONGJMP_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4227 if (RetVT.SimpleTy != MVT::isVoid)
4228 return Register();
4229 if ((Subtarget->isThumb()) && (Subtarget->isTargetWindows())) {
4230 return fastEmitInst_rr(MachineInstOpcode: ARM::tInt_WIN_eh_sjlj_longjmp, RC: &ARM::GPRRegClass, Op0, Op1);
4231 }
4232 if ((!Subtarget->isTargetWindows()) && (Subtarget->isThumb())) {
4233 return fastEmitInst_rr(MachineInstOpcode: ARM::tInt_eh_sjlj_longjmp, RC: &ARM::tGPRRegClass, Op0, Op1);
4234 }
4235 if ((!Subtarget->isThumb())) {
4236 return fastEmitInst_rr(MachineInstOpcode: ARM::Int_eh_sjlj_longjmp, RC: &ARM::GPRRegClass, Op0, Op1);
4237 }
4238 return Register();
4239}
4240
4241Register fastEmit_ARMISD_EH_SJLJ_LONGJMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4242 switch (VT.SimpleTy) {
4243 case MVT::i32: return fastEmit_ARMISD_EH_SJLJ_LONGJMP_MVT_i32_rr(RetVT, Op0, Op1);
4244 default: return Register();
4245 }
4246}
4247
4248// FastEmit functions for ARMISD::EH_SJLJ_SETJMP.
4249
4250Register fastEmit_ARMISD_EH_SJLJ_SETJMP_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4251 if (RetVT.SimpleTy != MVT::i32)
4252 return Register();
4253 if ((Subtarget->isThumb2()) && (!Subtarget->hasVFP2Base())) {
4254 return fastEmitInst_rr(MachineInstOpcode: ARM::t2Int_eh_sjlj_setjmp_nofp, RC: &ARM::tGPRRegClass, Op0, Op1);
4255 }
4256 if ((Subtarget->hasVFP2Base()) && (Subtarget->isThumb2())) {
4257 return fastEmitInst_rr(MachineInstOpcode: ARM::t2Int_eh_sjlj_setjmp, RC: &ARM::tGPRRegClass, Op0, Op1);
4258 }
4259 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
4260 return fastEmitInst_rr(MachineInstOpcode: ARM::tInt_eh_sjlj_setjmp, RC: &ARM::tGPRRegClass, Op0, Op1);
4261 }
4262 if ((!Subtarget->isThumb()) && (!Subtarget->hasVFP2Base())) {
4263 return fastEmitInst_rr(MachineInstOpcode: ARM::Int_eh_sjlj_setjmp_nofp, RC: &ARM::GPRRegClass, Op0, Op1);
4264 }
4265 if ((Subtarget->hasVFP2Base()) && (!Subtarget->isThumb())) {
4266 return fastEmitInst_rr(MachineInstOpcode: ARM::Int_eh_sjlj_setjmp, RC: &ARM::GPRRegClass, Op0, Op1);
4267 }
4268 return Register();
4269}
4270
4271Register fastEmit_ARMISD_EH_SJLJ_SETJMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4272 switch (VT.SimpleTy) {
4273 case MVT::i32: return fastEmit_ARMISD_EH_SJLJ_SETJMP_MVT_i32_rr(RetVT, Op0, Op1);
4274 default: return Register();
4275 }
4276}
4277
4278// FastEmit functions for ARMISD::QADD16b.
4279
4280Register fastEmit_ARMISD_QADD16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4281 if (RetVT.SimpleTy != MVT::i32)
4282 return Register();
4283 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4284 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QADD16, RC: &ARM::rGPRRegClass, Op0, Op1);
4285 }
4286 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4287 return fastEmitInst_rr(MachineInstOpcode: ARM::QADD16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4288 }
4289 return Register();
4290}
4291
4292Register fastEmit_ARMISD_QADD16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4293 switch (VT.SimpleTy) {
4294 case MVT::i32: return fastEmit_ARMISD_QADD16b_MVT_i32_rr(RetVT, Op0, Op1);
4295 default: return Register();
4296 }
4297}
4298
4299// FastEmit functions for ARMISD::QADD8b.
4300
4301Register fastEmit_ARMISD_QADD8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4302 if (RetVT.SimpleTy != MVT::i32)
4303 return Register();
4304 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4305 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QADD8, RC: &ARM::rGPRRegClass, Op0, Op1);
4306 }
4307 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4308 return fastEmitInst_rr(MachineInstOpcode: ARM::QADD8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4309 }
4310 return Register();
4311}
4312
4313Register fastEmit_ARMISD_QADD8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4314 switch (VT.SimpleTy) {
4315 case MVT::i32: return fastEmit_ARMISD_QADD8b_MVT_i32_rr(RetVT, Op0, Op1);
4316 default: return Register();
4317 }
4318}
4319
4320// FastEmit functions for ARMISD::QSUB16b.
4321
4322Register fastEmit_ARMISD_QSUB16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4323 if (RetVT.SimpleTy != MVT::i32)
4324 return Register();
4325 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4326 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QSUB16, RC: &ARM::rGPRRegClass, Op0, Op1);
4327 }
4328 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4329 return fastEmitInst_rr(MachineInstOpcode: ARM::QSUB16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4330 }
4331 return Register();
4332}
4333
4334Register fastEmit_ARMISD_QSUB16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4335 switch (VT.SimpleTy) {
4336 case MVT::i32: return fastEmit_ARMISD_QSUB16b_MVT_i32_rr(RetVT, Op0, Op1);
4337 default: return Register();
4338 }
4339}
4340
4341// FastEmit functions for ARMISD::QSUB8b.
4342
4343Register fastEmit_ARMISD_QSUB8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4344 if (RetVT.SimpleTy != MVT::i32)
4345 return Register();
4346 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4347 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QSUB8, RC: &ARM::rGPRRegClass, Op0, Op1);
4348 }
4349 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4350 return fastEmitInst_rr(MachineInstOpcode: ARM::QSUB8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4351 }
4352 return Register();
4353}
4354
4355Register fastEmit_ARMISD_QSUB8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4356 switch (VT.SimpleTy) {
4357 case MVT::i32: return fastEmit_ARMISD_QSUB8b_MVT_i32_rr(RetVT, Op0, Op1);
4358 default: return Register();
4359 }
4360}
4361
4362// FastEmit functions for ARMISD::SMULWB.
4363
4364Register fastEmit_ARMISD_SMULWB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4365 if (RetVT.SimpleTy != MVT::i32)
4366 return Register();
4367 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4368 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SMULWB, RC: &ARM::rGPRRegClass, Op0, Op1);
4369 }
4370 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
4371 return fastEmitInst_rr(MachineInstOpcode: ARM::SMULWB, RC: &ARM::GPRRegClass, Op0, Op1);
4372 }
4373 return Register();
4374}
4375
4376Register fastEmit_ARMISD_SMULWB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4377 switch (VT.SimpleTy) {
4378 case MVT::i32: return fastEmit_ARMISD_SMULWB_MVT_i32_rr(RetVT, Op0, Op1);
4379 default: return Register();
4380 }
4381}
4382
4383// FastEmit functions for ARMISD::SMULWT.
4384
4385Register fastEmit_ARMISD_SMULWT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4386 if (RetVT.SimpleTy != MVT::i32)
4387 return Register();
4388 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4389 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SMULWT, RC: &ARM::rGPRRegClass, Op0, Op1);
4390 }
4391 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
4392 return fastEmitInst_rr(MachineInstOpcode: ARM::SMULWT, RC: &ARM::GPRRegClass, Op0, Op1);
4393 }
4394 return Register();
4395}
4396
4397Register fastEmit_ARMISD_SMULWT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4398 switch (VT.SimpleTy) {
4399 case MVT::i32: return fastEmit_ARMISD_SMULWT_MVT_i32_rr(RetVT, Op0, Op1);
4400 default: return Register();
4401 }
4402}
4403
4404// FastEmit functions for ARMISD::UQADD16b.
4405
4406Register fastEmit_ARMISD_UQADD16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4407 if (RetVT.SimpleTy != MVT::i32)
4408 return Register();
4409 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4410 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQADD16, RC: &ARM::rGPRRegClass, Op0, Op1);
4411 }
4412 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4413 return fastEmitInst_rr(MachineInstOpcode: ARM::UQADD16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4414 }
4415 return Register();
4416}
4417
4418Register fastEmit_ARMISD_UQADD16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4419 switch (VT.SimpleTy) {
4420 case MVT::i32: return fastEmit_ARMISD_UQADD16b_MVT_i32_rr(RetVT, Op0, Op1);
4421 default: return Register();
4422 }
4423}
4424
4425// FastEmit functions for ARMISD::UQADD8b.
4426
4427Register fastEmit_ARMISD_UQADD8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4428 if (RetVT.SimpleTy != MVT::i32)
4429 return Register();
4430 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4431 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQADD8, RC: &ARM::rGPRRegClass, Op0, Op1);
4432 }
4433 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4434 return fastEmitInst_rr(MachineInstOpcode: ARM::UQADD8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4435 }
4436 return Register();
4437}
4438
4439Register fastEmit_ARMISD_UQADD8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4440 switch (VT.SimpleTy) {
4441 case MVT::i32: return fastEmit_ARMISD_UQADD8b_MVT_i32_rr(RetVT, Op0, Op1);
4442 default: return Register();
4443 }
4444}
4445
4446// FastEmit functions for ARMISD::UQSUB16b.
4447
4448Register fastEmit_ARMISD_UQSUB16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4449 if (RetVT.SimpleTy != MVT::i32)
4450 return Register();
4451 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4452 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQSUB16, RC: &ARM::rGPRRegClass, Op0, Op1);
4453 }
4454 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4455 return fastEmitInst_rr(MachineInstOpcode: ARM::UQSUB16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4456 }
4457 return Register();
4458}
4459
4460Register fastEmit_ARMISD_UQSUB16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4461 switch (VT.SimpleTy) {
4462 case MVT::i32: return fastEmit_ARMISD_UQSUB16b_MVT_i32_rr(RetVT, Op0, Op1);
4463 default: return Register();
4464 }
4465}
4466
4467// FastEmit functions for ARMISD::UQSUB8b.
4468
4469Register fastEmit_ARMISD_UQSUB8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4470 if (RetVT.SimpleTy != MVT::i32)
4471 return Register();
4472 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4473 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQSUB8, RC: &ARM::rGPRRegClass, Op0, Op1);
4474 }
4475 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4476 return fastEmitInst_rr(MachineInstOpcode: ARM::UQSUB8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4477 }
4478 return Register();
4479}
4480
4481Register fastEmit_ARMISD_UQSUB8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4482 switch (VT.SimpleTy) {
4483 case MVT::i32: return fastEmit_ARMISD_UQSUB8b_MVT_i32_rr(RetVT, Op0, Op1);
4484 default: return Register();
4485 }
4486}
4487
4488// FastEmit functions for ARMISD::VMLAVs.
4489
4490Register fastEmit_ARMISD_VMLAVs_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4491 if (RetVT.SimpleTy != MVT::i32)
4492 return Register();
4493 if ((Subtarget->hasMVEIntegerOps())) {
4494 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVs8, RC: &ARM::tGPREvenRegClass, Op0, Op1);
4495 }
4496 return Register();
4497}
4498
4499Register fastEmit_ARMISD_VMLAVs_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4500 if (RetVT.SimpleTy != MVT::i32)
4501 return Register();
4502 if ((Subtarget->hasMVEIntegerOps())) {
4503 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVs16, RC: &ARM::tGPREvenRegClass, Op0, Op1);
4504 }
4505 return Register();
4506}
4507
4508Register fastEmit_ARMISD_VMLAVs_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4509 switch (VT.SimpleTy) {
4510 case MVT::v16i8: return fastEmit_ARMISD_VMLAVs_MVT_v16i8_rr(RetVT, Op0, Op1);
4511 case MVT::v8i16: return fastEmit_ARMISD_VMLAVs_MVT_v8i16_rr(RetVT, Op0, Op1);
4512 default: return Register();
4513 }
4514}
4515
4516// FastEmit functions for ARMISD::VMLAVu.
4517
4518Register fastEmit_ARMISD_VMLAVu_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4519 if (RetVT.SimpleTy != MVT::i32)
4520 return Register();
4521 if ((Subtarget->hasMVEIntegerOps())) {
4522 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVu8, RC: &ARM::tGPREvenRegClass, Op0, Op1);
4523 }
4524 return Register();
4525}
4526
4527Register fastEmit_ARMISD_VMLAVu_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4528 if (RetVT.SimpleTy != MVT::i32)
4529 return Register();
4530 if ((Subtarget->hasMVEIntegerOps())) {
4531 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVu16, RC: &ARM::tGPREvenRegClass, Op0, Op1);
4532 }
4533 return Register();
4534}
4535
4536Register fastEmit_ARMISD_VMLAVu_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4537 switch (VT.SimpleTy) {
4538 case MVT::v16i8: return fastEmit_ARMISD_VMLAVu_MVT_v16i8_rr(RetVT, Op0, Op1);
4539 case MVT::v8i16: return fastEmit_ARMISD_VMLAVu_MVT_v8i16_rr(RetVT, Op0, Op1);
4540 default: return Register();
4541 }
4542}
4543
4544// FastEmit functions for ARMISD::VMOVDRR.
4545
4546Register fastEmit_ARMISD_VMOVDRR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4547 if (RetVT.SimpleTy != MVT::f64)
4548 return Register();
4549 if ((Subtarget->hasFPRegs())) {
4550 return fastEmitInst_rr(MachineInstOpcode: ARM::VMOVDRR, RC: &ARM::DPRRegClass, Op0, Op1);
4551 }
4552 return Register();
4553}
4554
4555Register fastEmit_ARMISD_VMOVDRR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4556 switch (VT.SimpleTy) {
4557 case MVT::i32: return fastEmit_ARMISD_VMOVDRR_MVT_i32_rr(RetVT, Op0, Op1);
4558 default: return Register();
4559 }
4560}
4561
4562// FastEmit functions for ARMISD::VMULLs.
4563
4564Register fastEmit_ARMISD_VMULLs_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4565 if (RetVT.SimpleTy != MVT::v8i16)
4566 return Register();
4567 if ((Subtarget->hasNEON())) {
4568 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4569 }
4570 return Register();
4571}
4572
4573Register fastEmit_ARMISD_VMULLs_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4574 if (RetVT.SimpleTy != MVT::v4i32)
4575 return Register();
4576 if ((Subtarget->hasNEON())) {
4577 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4578 }
4579 return Register();
4580}
4581
4582Register fastEmit_ARMISD_VMULLs_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4583 if (RetVT.SimpleTy != MVT::v2i64)
4584 return Register();
4585 if ((Subtarget->hasNEON())) {
4586 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4587 }
4588 return Register();
4589}
4590
4591Register fastEmit_ARMISD_VMULLs_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4592 if (RetVT.SimpleTy != MVT::v2i64)
4593 return Register();
4594 if ((Subtarget->hasMVEIntegerOps())) {
4595 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULLBs32, RC: &ARM::MQPRRegClass, Op0, Op1);
4596 }
4597 return Register();
4598}
4599
4600Register fastEmit_ARMISD_VMULLs_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4601 switch (VT.SimpleTy) {
4602 case MVT::v8i8: return fastEmit_ARMISD_VMULLs_MVT_v8i8_rr(RetVT, Op0, Op1);
4603 case MVT::v4i16: return fastEmit_ARMISD_VMULLs_MVT_v4i16_rr(RetVT, Op0, Op1);
4604 case MVT::v2i32: return fastEmit_ARMISD_VMULLs_MVT_v2i32_rr(RetVT, Op0, Op1);
4605 case MVT::v4i32: return fastEmit_ARMISD_VMULLs_MVT_v4i32_rr(RetVT, Op0, Op1);
4606 default: return Register();
4607 }
4608}
4609
4610// FastEmit functions for ARMISD::VMULLu.
4611
4612Register fastEmit_ARMISD_VMULLu_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4613 if (RetVT.SimpleTy != MVT::v8i16)
4614 return Register();
4615 if ((Subtarget->hasNEON())) {
4616 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4617 }
4618 return Register();
4619}
4620
4621Register fastEmit_ARMISD_VMULLu_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4622 if (RetVT.SimpleTy != MVT::v4i32)
4623 return Register();
4624 if ((Subtarget->hasNEON())) {
4625 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4626 }
4627 return Register();
4628}
4629
4630Register fastEmit_ARMISD_VMULLu_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4631 if (RetVT.SimpleTy != MVT::v2i64)
4632 return Register();
4633 if ((Subtarget->hasNEON())) {
4634 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4635 }
4636 return Register();
4637}
4638
4639Register fastEmit_ARMISD_VMULLu_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4640 if (RetVT.SimpleTy != MVT::v2i64)
4641 return Register();
4642 if ((Subtarget->hasMVEIntegerOps())) {
4643 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULLBu32, RC: &ARM::MQPRRegClass, Op0, Op1);
4644 }
4645 return Register();
4646}
4647
4648Register fastEmit_ARMISD_VMULLu_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4649 switch (VT.SimpleTy) {
4650 case MVT::v8i8: return fastEmit_ARMISD_VMULLu_MVT_v8i8_rr(RetVT, Op0, Op1);
4651 case MVT::v4i16: return fastEmit_ARMISD_VMULLu_MVT_v4i16_rr(RetVT, Op0, Op1);
4652 case MVT::v2i32: return fastEmit_ARMISD_VMULLu_MVT_v2i32_rr(RetVT, Op0, Op1);
4653 case MVT::v4i32: return fastEmit_ARMISD_VMULLu_MVT_v4i32_rr(RetVT, Op0, Op1);
4654 default: return Register();
4655 }
4656}
4657
4658// FastEmit functions for ARMISD::VQDMULH.
4659
4660Register fastEmit_ARMISD_VQDMULH_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4661 if (RetVT.SimpleTy != MVT::v16i8)
4662 return Register();
4663 if ((Subtarget->hasMVEIntegerOps())) {
4664 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQDMULHi8, RC: &ARM::MQPRRegClass, Op0, Op1);
4665 }
4666 return Register();
4667}
4668
4669Register fastEmit_ARMISD_VQDMULH_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4670 if (RetVT.SimpleTy != MVT::v8i16)
4671 return Register();
4672 if ((Subtarget->hasMVEIntegerOps())) {
4673 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQDMULHi16, RC: &ARM::MQPRRegClass, Op0, Op1);
4674 }
4675 return Register();
4676}
4677
4678Register fastEmit_ARMISD_VQDMULH_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4679 if (RetVT.SimpleTy != MVT::v4i32)
4680 return Register();
4681 if ((Subtarget->hasMVEIntegerOps())) {
4682 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQDMULHi32, RC: &ARM::MQPRRegClass, Op0, Op1);
4683 }
4684 return Register();
4685}
4686
4687Register fastEmit_ARMISD_VQDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4688 switch (VT.SimpleTy) {
4689 case MVT::v16i8: return fastEmit_ARMISD_VQDMULH_MVT_v16i8_rr(RetVT, Op0, Op1);
4690 case MVT::v8i16: return fastEmit_ARMISD_VQDMULH_MVT_v8i16_rr(RetVT, Op0, Op1);
4691 case MVT::v4i32: return fastEmit_ARMISD_VQDMULH_MVT_v4i32_rr(RetVT, Op0, Op1);
4692 default: return Register();
4693 }
4694}
4695
4696// FastEmit functions for ARMISD::VSHLs.
4697
4698Register fastEmit_ARMISD_VSHLs_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4699 if (RetVT.SimpleTy != MVT::v8i8)
4700 return Register();
4701 if ((Subtarget->hasNEON())) {
4702 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4703 }
4704 return Register();
4705}
4706
4707Register fastEmit_ARMISD_VSHLs_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4708 if (RetVT.SimpleTy != MVT::v16i8)
4709 return Register();
4710 if ((Subtarget->hasMVEIntegerOps())) {
4711 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecs8, RC: &ARM::MQPRRegClass, Op0, Op1);
4712 }
4713 if ((Subtarget->hasNEON())) {
4714 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4715 }
4716 return Register();
4717}
4718
4719Register fastEmit_ARMISD_VSHLs_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4720 if (RetVT.SimpleTy != MVT::v4i16)
4721 return Register();
4722 if ((Subtarget->hasNEON())) {
4723 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4724 }
4725 return Register();
4726}
4727
4728Register fastEmit_ARMISD_VSHLs_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4729 if (RetVT.SimpleTy != MVT::v8i16)
4730 return Register();
4731 if ((Subtarget->hasMVEIntegerOps())) {
4732 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecs16, RC: &ARM::MQPRRegClass, Op0, Op1);
4733 }
4734 if ((Subtarget->hasNEON())) {
4735 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4736 }
4737 return Register();
4738}
4739
4740Register fastEmit_ARMISD_VSHLs_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4741 if (RetVT.SimpleTy != MVT::v2i32)
4742 return Register();
4743 if ((Subtarget->hasNEON())) {
4744 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4745 }
4746 return Register();
4747}
4748
4749Register fastEmit_ARMISD_VSHLs_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4750 if (RetVT.SimpleTy != MVT::v4i32)
4751 return Register();
4752 if ((Subtarget->hasMVEIntegerOps())) {
4753 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecs32, RC: &ARM::MQPRRegClass, Op0, Op1);
4754 }
4755 if ((Subtarget->hasNEON())) {
4756 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4757 }
4758 return Register();
4759}
4760
4761Register fastEmit_ARMISD_VSHLs_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
4762 if (RetVT.SimpleTy != MVT::v1i64)
4763 return Register();
4764 if ((Subtarget->hasNEON())) {
4765 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
4766 }
4767 return Register();
4768}
4769
4770Register fastEmit_ARMISD_VSHLs_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
4771 if (RetVT.SimpleTy != MVT::v2i64)
4772 return Register();
4773 if ((Subtarget->hasNEON())) {
4774 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4775 }
4776 return Register();
4777}
4778
4779Register fastEmit_ARMISD_VSHLs_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4780 switch (VT.SimpleTy) {
4781 case MVT::v8i8: return fastEmit_ARMISD_VSHLs_MVT_v8i8_rr(RetVT, Op0, Op1);
4782 case MVT::v16i8: return fastEmit_ARMISD_VSHLs_MVT_v16i8_rr(RetVT, Op0, Op1);
4783 case MVT::v4i16: return fastEmit_ARMISD_VSHLs_MVT_v4i16_rr(RetVT, Op0, Op1);
4784 case MVT::v8i16: return fastEmit_ARMISD_VSHLs_MVT_v8i16_rr(RetVT, Op0, Op1);
4785 case MVT::v2i32: return fastEmit_ARMISD_VSHLs_MVT_v2i32_rr(RetVT, Op0, Op1);
4786 case MVT::v4i32: return fastEmit_ARMISD_VSHLs_MVT_v4i32_rr(RetVT, Op0, Op1);
4787 case MVT::v1i64: return fastEmit_ARMISD_VSHLs_MVT_v1i64_rr(RetVT, Op0, Op1);
4788 case MVT::v2i64: return fastEmit_ARMISD_VSHLs_MVT_v2i64_rr(RetVT, Op0, Op1);
4789 default: return Register();
4790 }
4791}
4792
4793// FastEmit functions for ARMISD::VSHLu.
4794
4795Register fastEmit_ARMISD_VSHLu_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4796 if (RetVT.SimpleTy != MVT::v8i8)
4797 return Register();
4798 if ((Subtarget->hasNEON())) {
4799 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4800 }
4801 return Register();
4802}
4803
4804Register fastEmit_ARMISD_VSHLu_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4805 if (RetVT.SimpleTy != MVT::v16i8)
4806 return Register();
4807 if ((Subtarget->hasMVEIntegerOps())) {
4808 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecu8, RC: &ARM::MQPRRegClass, Op0, Op1);
4809 }
4810 if ((Subtarget->hasNEON())) {
4811 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4812 }
4813 return Register();
4814}
4815
4816Register fastEmit_ARMISD_VSHLu_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4817 if (RetVT.SimpleTy != MVT::v4i16)
4818 return Register();
4819 if ((Subtarget->hasNEON())) {
4820 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4821 }
4822 return Register();
4823}
4824
4825Register fastEmit_ARMISD_VSHLu_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4826 if (RetVT.SimpleTy != MVT::v8i16)
4827 return Register();
4828 if ((Subtarget->hasMVEIntegerOps())) {
4829 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecu16, RC: &ARM::MQPRRegClass, Op0, Op1);
4830 }
4831 if ((Subtarget->hasNEON())) {
4832 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4833 }
4834 return Register();
4835}
4836
4837Register fastEmit_ARMISD_VSHLu_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4838 if (RetVT.SimpleTy != MVT::v2i32)
4839 return Register();
4840 if ((Subtarget->hasNEON())) {
4841 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4842 }
4843 return Register();
4844}
4845
4846Register fastEmit_ARMISD_VSHLu_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4847 if (RetVT.SimpleTy != MVT::v4i32)
4848 return Register();
4849 if ((Subtarget->hasMVEIntegerOps())) {
4850 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecu32, RC: &ARM::MQPRRegClass, Op0, Op1);
4851 }
4852 if ((Subtarget->hasNEON())) {
4853 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4854 }
4855 return Register();
4856}
4857
4858Register fastEmit_ARMISD_VSHLu_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
4859 if (RetVT.SimpleTy != MVT::v1i64)
4860 return Register();
4861 if ((Subtarget->hasNEON())) {
4862 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
4863 }
4864 return Register();
4865}
4866
4867Register fastEmit_ARMISD_VSHLu_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
4868 if (RetVT.SimpleTy != MVT::v2i64)
4869 return Register();
4870 if ((Subtarget->hasNEON())) {
4871 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4872 }
4873 return Register();
4874}
4875
4876Register fastEmit_ARMISD_VSHLu_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4877 switch (VT.SimpleTy) {
4878 case MVT::v8i8: return fastEmit_ARMISD_VSHLu_MVT_v8i8_rr(RetVT, Op0, Op1);
4879 case MVT::v16i8: return fastEmit_ARMISD_VSHLu_MVT_v16i8_rr(RetVT, Op0, Op1);
4880 case MVT::v4i16: return fastEmit_ARMISD_VSHLu_MVT_v4i16_rr(RetVT, Op0, Op1);
4881 case MVT::v8i16: return fastEmit_ARMISD_VSHLu_MVT_v8i16_rr(RetVT, Op0, Op1);
4882 case MVT::v2i32: return fastEmit_ARMISD_VSHLu_MVT_v2i32_rr(RetVT, Op0, Op1);
4883 case MVT::v4i32: return fastEmit_ARMISD_VSHLu_MVT_v4i32_rr(RetVT, Op0, Op1);
4884 case MVT::v1i64: return fastEmit_ARMISD_VSHLu_MVT_v1i64_rr(RetVT, Op0, Op1);
4885 case MVT::v2i64: return fastEmit_ARMISD_VSHLu_MVT_v2i64_rr(RetVT, Op0, Op1);
4886 default: return Register();
4887 }
4888}
4889
4890// FastEmit functions for ARMISD::VTBL1.
4891
4892Register fastEmit_ARMISD_VTBL1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4893 if (RetVT.SimpleTy != MVT::v8i8)
4894 return Register();
4895 if ((Subtarget->hasNEON())) {
4896 return fastEmitInst_rr(MachineInstOpcode: ARM::VTBL1, RC: &ARM::DPRRegClass, Op0, Op1);
4897 }
4898 return Register();
4899}
4900
4901Register fastEmit_ARMISD_VTBL1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4902 switch (VT.SimpleTy) {
4903 case MVT::v8i8: return fastEmit_ARMISD_VTBL1_MVT_v8i8_rr(RetVT, Op0, Op1);
4904 default: return Register();
4905 }
4906}
4907
4908// FastEmit functions for ARMISD::VTST.
4909
4910Register fastEmit_ARMISD_VTST_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4911 if (RetVT.SimpleTy != MVT::v8i8)
4912 return Register();
4913 if ((Subtarget->hasNEON())) {
4914 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4915 }
4916 return Register();
4917}
4918
4919Register fastEmit_ARMISD_VTST_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4920 if (RetVT.SimpleTy != MVT::v16i8)
4921 return Register();
4922 if ((Subtarget->hasNEON())) {
4923 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4924 }
4925 return Register();
4926}
4927
4928Register fastEmit_ARMISD_VTST_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4929 if (RetVT.SimpleTy != MVT::v4i16)
4930 return Register();
4931 if ((Subtarget->hasNEON())) {
4932 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4933 }
4934 return Register();
4935}
4936
4937Register fastEmit_ARMISD_VTST_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4938 if (RetVT.SimpleTy != MVT::v8i16)
4939 return Register();
4940 if ((Subtarget->hasNEON())) {
4941 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4942 }
4943 return Register();
4944}
4945
4946Register fastEmit_ARMISD_VTST_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4947 if (RetVT.SimpleTy != MVT::v2i32)
4948 return Register();
4949 if ((Subtarget->hasNEON())) {
4950 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4951 }
4952 return Register();
4953}
4954
4955Register fastEmit_ARMISD_VTST_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4956 if (RetVT.SimpleTy != MVT::v4i32)
4957 return Register();
4958 if ((Subtarget->hasNEON())) {
4959 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4960 }
4961 return Register();
4962}
4963
4964Register fastEmit_ARMISD_VTST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4965 switch (VT.SimpleTy) {
4966 case MVT::v8i8: return fastEmit_ARMISD_VTST_MVT_v8i8_rr(RetVT, Op0, Op1);
4967 case MVT::v16i8: return fastEmit_ARMISD_VTST_MVT_v16i8_rr(RetVT, Op0, Op1);
4968 case MVT::v4i16: return fastEmit_ARMISD_VTST_MVT_v4i16_rr(RetVT, Op0, Op1);
4969 case MVT::v8i16: return fastEmit_ARMISD_VTST_MVT_v8i16_rr(RetVT, Op0, Op1);
4970 case MVT::v2i32: return fastEmit_ARMISD_VTST_MVT_v2i32_rr(RetVT, Op0, Op1);
4971 case MVT::v4i32: return fastEmit_ARMISD_VTST_MVT_v4i32_rr(RetVT, Op0, Op1);
4972 default: return Register();
4973 }
4974}
4975
4976// FastEmit functions for ISD::ABDS.
4977
4978Register fastEmit_ISD_ABDS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4979 if (RetVT.SimpleTy != MVT::v8i8)
4980 return Register();
4981 if ((Subtarget->hasNEON())) {
4982 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4983 }
4984 return Register();
4985}
4986
4987Register fastEmit_ISD_ABDS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4988 if (RetVT.SimpleTy != MVT::v16i8)
4989 return Register();
4990 if ((Subtarget->hasMVEIntegerOps())) {
4991 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
4992 }
4993 if ((Subtarget->hasNEON())) {
4994 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4995 }
4996 return Register();
4997}
4998
4999Register fastEmit_ISD_ABDS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
5000 if (RetVT.SimpleTy != MVT::v4i16)
5001 return Register();
5002 if ((Subtarget->hasNEON())) {
5003 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
5004 }
5005 return Register();
5006}
5007
5008Register fastEmit_ISD_ABDS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5009 if (RetVT.SimpleTy != MVT::v8i16)
5010 return Register();
5011 if ((Subtarget->hasMVEIntegerOps())) {
5012 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
5013 }
5014 if ((Subtarget->hasNEON())) {
5015 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
5016 }
5017 return Register();
5018}
5019
5020Register fastEmit_ISD_ABDS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
5021 if (RetVT.SimpleTy != MVT::v2i32)
5022 return Register();
5023 if ((Subtarget->hasNEON())) {
5024 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
5025 }
5026 return Register();
5027}
5028
5029Register fastEmit_ISD_ABDS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5030 if (RetVT.SimpleTy != MVT::v4i32)
5031 return Register();
5032 if ((Subtarget->hasMVEIntegerOps())) {
5033 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
5034 }
5035 if ((Subtarget->hasNEON())) {
5036 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
5037 }
5038 return Register();
5039}
5040
5041Register fastEmit_ISD_ABDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5042 switch (VT.SimpleTy) {
5043 case MVT::v8i8: return fastEmit_ISD_ABDS_MVT_v8i8_rr(RetVT, Op0, Op1);
5044 case MVT::v16i8: return fastEmit_ISD_ABDS_MVT_v16i8_rr(RetVT, Op0, Op1);
5045 case MVT::v4i16: return fastEmit_ISD_ABDS_MVT_v4i16_rr(RetVT, Op0, Op1);
5046 case MVT::v8i16: return fastEmit_ISD_ABDS_MVT_v8i16_rr(RetVT, Op0, Op1);
5047 case MVT::v2i32: return fastEmit_ISD_ABDS_MVT_v2i32_rr(RetVT, Op0, Op1);
5048 case MVT::v4i32: return fastEmit_ISD_ABDS_MVT_v4i32_rr(RetVT, Op0, Op1);
5049 default: return Register();
5050 }
5051}
5052
5053// FastEmit functions for ISD::ABDU.
5054
5055Register fastEmit_ISD_ABDU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
5056 if (RetVT.SimpleTy != MVT::v8i8)
5057 return Register();
5058 if ((Subtarget->hasNEON())) {
5059 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
5060 }
5061 return Register();
5062}
5063
5064Register fastEmit_ISD_ABDU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5065 if (RetVT.SimpleTy != MVT::v16i8)
5066 return Register();
5067 if ((Subtarget->hasMVEIntegerOps())) {
5068 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
5069 }
5070 if ((Subtarget->hasNEON())) {
5071 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
5072 }
5073 return Register();
5074}
5075
5076Register fastEmit_ISD_ABDU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
5077 if (RetVT.SimpleTy != MVT::v4i16)
5078 return Register();
5079 if ((Subtarget->hasNEON())) {
5080 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
5081 }
5082 return Register();
5083}
5084
5085Register fastEmit_ISD_ABDU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5086 if (RetVT.SimpleTy != MVT::v8i16)
5087 return Register();
5088 if ((Subtarget->hasMVEIntegerOps())) {
5089 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
5090 }
5091 if ((Subtarget->hasNEON())) {
5092 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
5093 }
5094 return Register();
5095}
5096
5097Register fastEmit_ISD_ABDU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
5098 if (RetVT.SimpleTy != MVT::v2i32)
5099 return Register();
5100 if ((Subtarget->hasNEON())) {
5101 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
5102 }
5103 return Register();
5104}
5105
5106Register fastEmit_ISD_ABDU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5107 if (RetVT.SimpleTy != MVT::v4i32)
5108 return Register();
5109 if ((Subtarget->hasMVEIntegerOps())) {
5110 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
5111 }
5112 if ((Subtarget->hasNEON())) {
5113 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
5114 }
5115 return Register();
5116}
5117
5118Register fastEmit_ISD_ABDU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5119 switch (VT.SimpleTy) {
5120 case MVT::v8i8: return fastEmit_ISD_ABDU_MVT_v8i8_rr(RetVT, Op0, Op1);
5121 case MVT::v16i8: return fastEmit_ISD_ABDU_MVT_v16i8_rr(RetVT, Op0, Op1);
5122 case MVT::v4i16: return fastEmit_ISD_ABDU_MVT_v4i16_rr(RetVT, Op0, Op1);
5123 case MVT::v8i16: return fastEmit_ISD_ABDU_MVT_v8i16_rr(RetVT, Op0, Op1);
5124 case MVT::v2i32: return fastEmit_ISD_ABDU_MVT_v2i32_rr(RetVT, Op0, Op1);
5125 case MVT::v4i32: return fastEmit_ISD_ABDU_MVT_v4i32_rr(RetVT, Op0, Op1);
5126 default: return Register();
5127 }
5128}
5129
5130// FastEmit functions for ISD::ADD.
5131
5132Register fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5133 if (RetVT.SimpleTy != MVT::i32)
5134 return Register();
5135 if ((Subtarget->isThumb2())) {
5136 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ADDrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
5137 }
5138 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5139 return fastEmitInst_rr(MachineInstOpcode: ARM::tADDrr, RC: &ARM::tGPRRegClass, Op0, Op1);
5140 }
5141 if ((!Subtarget->isThumb())) {
5142 return fastEmitInst_rr(MachineInstOpcode: ARM::ADDrr, RC: &ARM::GPRRegClass, Op0, Op1);
5143 }
5144 return Register();
5145}
5146
5147Register fastEmit_ISD_ADD_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
5148 if (RetVT.SimpleTy != MVT::v8i8)
5149 return Register();
5150 if ((Subtarget->hasNEON())) {
5151 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
5152 }
5153 return Register();
5154}
5155
5156Register fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5157 if (RetVT.SimpleTy != MVT::v16i8)
5158 return Register();
5159 if ((Subtarget->hasMVEIntegerOps())) {
5160 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDi8, RC: &ARM::MQPRRegClass, Op0, Op1);
5161 }
5162 if ((Subtarget->hasNEON())) {
5163 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
5164 }
5165 return Register();
5166}
5167
5168Register fastEmit_ISD_ADD_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
5169 if (RetVT.SimpleTy != MVT::v4i16)
5170 return Register();
5171 if ((Subtarget->hasNEON())) {
5172 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
5173 }
5174 return Register();
5175}
5176
5177Register fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5178 if (RetVT.SimpleTy != MVT::v8i16)
5179 return Register();
5180 if ((Subtarget->hasMVEIntegerOps())) {
5181 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDi16, RC: &ARM::MQPRRegClass, Op0, Op1);
5182 }
5183 if ((Subtarget->hasNEON())) {
5184 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
5185 }
5186 return Register();
5187}
5188
5189Register fastEmit_ISD_ADD_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
5190 if (RetVT.SimpleTy != MVT::v2i32)
5191 return Register();
5192 if ((Subtarget->hasNEON())) {
5193 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
5194 }
5195 return Register();
5196}
5197
5198Register fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5199 if (RetVT.SimpleTy != MVT::v4i32)
5200 return Register();
5201 if ((Subtarget->hasMVEIntegerOps())) {
5202 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDi32, RC: &ARM::MQPRRegClass, Op0, Op1);
5203 }
5204 if ((Subtarget->hasNEON())) {
5205 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
5206 }
5207 return Register();
5208}
5209
5210Register fastEmit_ISD_ADD_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
5211 if (RetVT.SimpleTy != MVT::v1i64)
5212 return Register();
5213 if ((Subtarget->hasNEON())) {
5214 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
5215 }
5216 return Register();
5217}
5218
5219Register fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
5220 if (RetVT.SimpleTy != MVT::v2i64)
5221 return Register();
5222 if ((Subtarget->hasNEON())) {
5223 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
5224 }
5225 return Register();
5226}
5227
5228Register fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5229 switch (VT.SimpleTy) {
5230 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
5231 case MVT::v8i8: return fastEmit_ISD_ADD_MVT_v8i8_rr(RetVT, Op0, Op1);
5232 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
5233 case MVT::v4i16: return fastEmit_ISD_ADD_MVT_v4i16_rr(RetVT, Op0, Op1);
5234 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
5235 case MVT::v2i32: return fastEmit_ISD_ADD_MVT_v2i32_rr(RetVT, Op0, Op1);
5236 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
5237 case MVT::v1i64: return fastEmit_ISD_ADD_MVT_v1i64_rr(RetVT, Op0, Op1);
5238 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
5239 default: return Register();
5240 }
5241}
5242
5243// FastEmit functions for ISD::AND.
5244
5245Register fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5246 if (RetVT.SimpleTy != MVT::i32)
5247 return Register();
5248 if ((Subtarget->isThumb2())) {
5249 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ANDrr, RC: &ARM::rGPRRegClass, Op0, Op1);
5250 }
5251 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5252 return fastEmitInst_rr(MachineInstOpcode: ARM::tAND, RC: &ARM::tGPRRegClass, Op0, Op1);
5253 }
5254 if ((!Subtarget->isThumb())) {
5255 return fastEmitInst_rr(MachineInstOpcode: ARM::ANDrr, RC: &ARM::GPRRegClass, Op0, Op1);
5256 }
5257 return Register();
5258}
5259
5260Register fastEmit_ISD_AND_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
5261 if (RetVT.SimpleTy != MVT::v8i8)
5262 return Register();
5263 if ((Subtarget->hasNEON())) {
5264 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
5265 }
5266 return Register();
5267}
5268
5269Register fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5270 if (RetVT.SimpleTy != MVT::v16i8)
5271 return Register();
5272 if ((Subtarget->hasMVEIntegerOps())) {
5273 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
5274 }
5275 if ((Subtarget->hasNEON())) {
5276 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
5277 }
5278 return Register();
5279}
5280
5281Register fastEmit_ISD_AND_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
5282 if (RetVT.SimpleTy != MVT::v4i16)
5283 return Register();
5284 if ((Subtarget->hasNEON())) {
5285 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
5286 }
5287 return Register();
5288}
5289
5290Register fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5291 if (RetVT.SimpleTy != MVT::v8i16)
5292 return Register();
5293 if ((Subtarget->hasMVEIntegerOps())) {
5294 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
5295 }
5296 if ((Subtarget->hasNEON())) {
5297 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
5298 }
5299 return Register();
5300}
5301
5302Register fastEmit_ISD_AND_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
5303 if (RetVT.SimpleTy != MVT::v2i32)
5304 return Register();
5305 if ((Subtarget->hasNEON())) {
5306 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
5307 }
5308 return Register();
5309}
5310
5311Register fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5312 if (RetVT.SimpleTy != MVT::v4i32)
5313 return Register();
5314 if ((Subtarget->hasMVEIntegerOps())) {
5315 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
5316 }
5317 if ((Subtarget->hasNEON())) {
5318 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
5319 }
5320 return Register();
5321}
5322
5323Register fastEmit_ISD_AND_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
5324 if (RetVT.SimpleTy != MVT::v1i64)
5325 return Register();
5326 if ((Subtarget->hasNEON())) {
5327 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
5328 }
5329 return Register();
5330}
5331
5332Register fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
5333 if (RetVT.SimpleTy != MVT::v2i64)
5334 return Register();
5335 if ((Subtarget->hasMVEIntegerOps())) {
5336 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
5337 }
5338 if ((Subtarget->hasNEON())) {
5339 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
5340 }
5341 return Register();
5342}
5343
5344Register fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5345 switch (VT.SimpleTy) {
5346 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
5347 case MVT::v8i8: return fastEmit_ISD_AND_MVT_v8i8_rr(RetVT, Op0, Op1);
5348 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
5349 case MVT::v4i16: return fastEmit_ISD_AND_MVT_v4i16_rr(RetVT, Op0, Op1);
5350 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
5351 case MVT::v2i32: return fastEmit_ISD_AND_MVT_v2i32_rr(RetVT, Op0, Op1);
5352 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
5353 case MVT::v1i64: return fastEmit_ISD_AND_MVT_v1i64_rr(RetVT, Op0, Op1);
5354 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
5355 default: return Register();
5356 }
5357}
5358
5359// FastEmit functions for ISD::AVGCEILS.
5360
5361Register fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5362 if (RetVT.SimpleTy != MVT::v16i8)
5363 return Register();
5364 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
5365}
5366
5367Register fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5368 if (RetVT.SimpleTy != MVT::v8i16)
5369 return Register();
5370 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
5371}
5372
5373Register fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5374 if (RetVT.SimpleTy != MVT::v4i32)
5375 return Register();
5376 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
5377}
5378
5379Register fastEmit_ISD_AVGCEILS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5380 switch (VT.SimpleTy) {
5381 case MVT::v16i8: return fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(RetVT, Op0, Op1);
5382 case MVT::v8i16: return fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(RetVT, Op0, Op1);
5383 case MVT::v4i32: return fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(RetVT, Op0, Op1);
5384 default: return Register();
5385 }
5386}
5387
5388// FastEmit functions for ISD::AVGCEILU.
5389
5390Register fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5391 if (RetVT.SimpleTy != MVT::v16i8)
5392 return Register();
5393 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
5394}
5395
5396Register fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5397 if (RetVT.SimpleTy != MVT::v8i16)
5398 return Register();
5399 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
5400}
5401
5402Register fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5403 if (RetVT.SimpleTy != MVT::v4i32)
5404 return Register();
5405 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
5406}
5407
5408Register fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5409 switch (VT.SimpleTy) {
5410 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
5411 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
5412 case MVT::v4i32: return fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(RetVT, Op0, Op1);
5413 default: return Register();
5414 }
5415}
5416
5417// FastEmit functions for ISD::AVGFLOORS.
5418
5419Register fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5420 if (RetVT.SimpleTy != MVT::v16i8)
5421 return Register();
5422 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
5423}
5424
5425Register fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5426 if (RetVT.SimpleTy != MVT::v8i16)
5427 return Register();
5428 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
5429}
5430
5431Register fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5432 if (RetVT.SimpleTy != MVT::v4i32)
5433 return Register();
5434 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
5435}
5436
5437Register fastEmit_ISD_AVGFLOORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5438 switch (VT.SimpleTy) {
5439 case MVT::v16i8: return fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(RetVT, Op0, Op1);
5440 case MVT::v8i16: return fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(RetVT, Op0, Op1);
5441 case MVT::v4i32: return fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(RetVT, Op0, Op1);
5442 default: return Register();
5443 }
5444}
5445
5446// FastEmit functions for ISD::AVGFLOORU.
5447
5448Register fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5449 if (RetVT.SimpleTy != MVT::v16i8)
5450 return Register();
5451 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
5452}
5453
5454Register fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5455 if (RetVT.SimpleTy != MVT::v8i16)
5456 return Register();
5457 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
5458}
5459
5460Register fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5461 if (RetVT.SimpleTy != MVT::v4i32)
5462 return Register();
5463 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
5464}
5465
5466Register fastEmit_ISD_AVGFLOORU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5467 switch (VT.SimpleTy) {
5468 case MVT::v16i8: return fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(RetVT, Op0, Op1);
5469 case MVT::v8i16: return fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(RetVT, Op0, Op1);
5470 case MVT::v4i32: return fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(RetVT, Op0, Op1);
5471 default: return Register();
5472 }
5473}
5474
5475// FastEmit functions for ISD::FADD.
5476
5477Register fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5478 if (RetVT.SimpleTy != MVT::f16)
5479 return Register();
5480 if ((Subtarget->hasFullFP16())) {
5481 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDH, RC: &ARM::HPRRegClass, Op0, Op1);
5482 }
5483 return Register();
5484}
5485
5486Register fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5487 if (RetVT.SimpleTy != MVT::f32)
5488 return Register();
5489 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
5490 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDS, RC: &ARM::SPRRegClass, Op0, Op1);
5491 }
5492 return Register();
5493}
5494
5495Register fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5496 if (RetVT.SimpleTy != MVT::f64)
5497 return Register();
5498 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
5499 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDD, RC: &ARM::DPRRegClass, Op0, Op1);
5500 }
5501 return Register();
5502}
5503
5504Register fastEmit_ISD_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5505 if (RetVT.SimpleTy != MVT::v4f16)
5506 return Register();
5507 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5508 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDhd, RC: &ARM::DPRRegClass, Op0, Op1);
5509 }
5510 return Register();
5511}
5512
5513Register fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5514 if (RetVT.SimpleTy != MVT::v8f16)
5515 return Register();
5516 if ((Subtarget->hasMVEFloatOps())) {
5517 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5518 }
5519 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5520 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDhq, RC: &ARM::QPRRegClass, Op0, Op1);
5521 }
5522 return Register();
5523}
5524
5525Register fastEmit_ISD_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5526 if (RetVT.SimpleTy != MVT::v2f32)
5527 return Register();
5528 if ((Subtarget->hasNEON())) {
5529 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDfd, RC: &ARM::DPRRegClass, Op0, Op1);
5530 }
5531 return Register();
5532}
5533
5534Register fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5535 if (RetVT.SimpleTy != MVT::v4f32)
5536 return Register();
5537 if ((Subtarget->hasMVEFloatOps())) {
5538 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5539 }
5540 if ((Subtarget->hasNEON())) {
5541 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDfq, RC: &ARM::QPRRegClass, Op0, Op1);
5542 }
5543 return Register();
5544}
5545
5546Register fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5547 switch (VT.SimpleTy) {
5548 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
5549 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
5550 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
5551 case MVT::v4f16: return fastEmit_ISD_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
5552 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
5553 case MVT::v2f32: return fastEmit_ISD_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
5554 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
5555 default: return Register();
5556 }
5557}
5558
5559// FastEmit functions for ISD::FDIV.
5560
5561Register fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5562 if (RetVT.SimpleTy != MVT::f16)
5563 return Register();
5564 if ((Subtarget->hasFullFP16())) {
5565 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVH, RC: &ARM::HPRRegClass, Op0, Op1);
5566 }
5567 return Register();
5568}
5569
5570Register fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5571 if (RetVT.SimpleTy != MVT::f32)
5572 return Register();
5573 if ((Subtarget->hasVFP2Base())) {
5574 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVS, RC: &ARM::SPRRegClass, Op0, Op1);
5575 }
5576 return Register();
5577}
5578
5579Register fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5580 if (RetVT.SimpleTy != MVT::f64)
5581 return Register();
5582 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
5583 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVD, RC: &ARM::DPRRegClass, Op0, Op1);
5584 }
5585 return Register();
5586}
5587
5588Register fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5589 switch (VT.SimpleTy) {
5590 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
5591 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
5592 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
5593 default: return Register();
5594 }
5595}
5596
5597// FastEmit functions for ISD::FMAXIMUM.
5598
5599Register fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5600 if (RetVT.SimpleTy != MVT::v4f16)
5601 return Register();
5602 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5603 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXhd, RC: &ARM::DPRRegClass, Op0, Op1);
5604 }
5605 return Register();
5606}
5607
5608Register fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5609 if (RetVT.SimpleTy != MVT::v8f16)
5610 return Register();
5611 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5612 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXhq, RC: &ARM::QPRRegClass, Op0, Op1);
5613 }
5614 return Register();
5615}
5616
5617Register fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5618 if (RetVT.SimpleTy != MVT::v2f32)
5619 return Register();
5620 if ((Subtarget->hasNEON())) {
5621 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXfd, RC: &ARM::DPRRegClass, Op0, Op1);
5622 }
5623 return Register();
5624}
5625
5626Register fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5627 if (RetVT.SimpleTy != MVT::v4f32)
5628 return Register();
5629 if ((Subtarget->hasNEON())) {
5630 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXfq, RC: &ARM::QPRRegClass, Op0, Op1);
5631 }
5632 return Register();
5633}
5634
5635Register fastEmit_ISD_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5636 switch (VT.SimpleTy) {
5637 case MVT::v4f16: return fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
5638 case MVT::v8f16: return fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
5639 case MVT::v2f32: return fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
5640 case MVT::v4f32: return fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
5641 default: return Register();
5642 }
5643}
5644
5645// FastEmit functions for ISD::FMAXNUM.
5646
5647Register fastEmit_ISD_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5648 if (RetVT.SimpleTy != MVT::f16)
5649 return Register();
5650 if ((Subtarget->hasFullFP16())) {
5651 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMH, RC: &ARM::HPRRegClass, Op0, Op1);
5652 }
5653 return Register();
5654}
5655
5656Register fastEmit_ISD_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5657 if (RetVT.SimpleTy != MVT::f32)
5658 return Register();
5659 if ((Subtarget->hasFPARMv8Base())) {
5660 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMS, RC: &ARM::SPRRegClass, Op0, Op1);
5661 }
5662 return Register();
5663}
5664
5665Register fastEmit_ISD_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5666 if (RetVT.SimpleTy != MVT::f64)
5667 return Register();
5668 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
5669 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMD, RC: &ARM::DPRRegClass, Op0, Op1);
5670 }
5671 return Register();
5672}
5673
5674Register fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5675 if (RetVT.SimpleTy != MVT::v4f16)
5676 return Register();
5677 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5678 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNDh, RC: &ARM::DPRRegClass, Op0, Op1);
5679 }
5680 return Register();
5681}
5682
5683Register fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5684 if (RetVT.SimpleTy != MVT::v8f16)
5685 return Register();
5686 if ((Subtarget->hasMVEFloatOps())) {
5687 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXNMf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5688 }
5689 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5690 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNQh, RC: &ARM::QPRRegClass, Op0, Op1);
5691 }
5692 return Register();
5693}
5694
5695Register fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5696 if (RetVT.SimpleTy != MVT::v2f32)
5697 return Register();
5698 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
5699 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNDf, RC: &ARM::DPRRegClass, Op0, Op1);
5700 }
5701 return Register();
5702}
5703
5704Register fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5705 if (RetVT.SimpleTy != MVT::v4f32)
5706 return Register();
5707 if ((Subtarget->hasMVEFloatOps())) {
5708 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXNMf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5709 }
5710 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
5711 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNQf, RC: &ARM::QPRRegClass, Op0, Op1);
5712 }
5713 return Register();
5714}
5715
5716Register fastEmit_ISD_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5717 switch (VT.SimpleTy) {
5718 case MVT::f16: return fastEmit_ISD_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
5719 case MVT::f32: return fastEmit_ISD_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
5720 case MVT::f64: return fastEmit_ISD_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
5721 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
5722 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
5723 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
5724 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
5725 default: return Register();
5726 }
5727}
5728
5729// FastEmit functions for ISD::FMINIMUM.
5730
5731Register fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5732 if (RetVT.SimpleTy != MVT::v4f16)
5733 return Register();
5734 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5735 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINhd, RC: &ARM::DPRRegClass, Op0, Op1);
5736 }
5737 return Register();
5738}
5739
5740Register fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5741 if (RetVT.SimpleTy != MVT::v8f16)
5742 return Register();
5743 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5744 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINhq, RC: &ARM::QPRRegClass, Op0, Op1);
5745 }
5746 return Register();
5747}
5748
5749Register fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5750 if (RetVT.SimpleTy != MVT::v2f32)
5751 return Register();
5752 if ((Subtarget->hasNEON())) {
5753 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINfd, RC: &ARM::DPRRegClass, Op0, Op1);
5754 }
5755 return Register();
5756}
5757
5758Register fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5759 if (RetVT.SimpleTy != MVT::v4f32)
5760 return Register();
5761 if ((Subtarget->hasNEON())) {
5762 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINfq, RC: &ARM::QPRRegClass, Op0, Op1);
5763 }
5764 return Register();
5765}
5766
5767Register fastEmit_ISD_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5768 switch (VT.SimpleTy) {
5769 case MVT::v4f16: return fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
5770 case MVT::v8f16: return fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
5771 case MVT::v2f32: return fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
5772 case MVT::v4f32: return fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
5773 default: return Register();
5774 }
5775}
5776
5777// FastEmit functions for ISD::FMINNUM.
5778
5779Register fastEmit_ISD_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5780 if (RetVT.SimpleTy != MVT::f16)
5781 return Register();
5782 if ((Subtarget->hasFullFP16())) {
5783 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMH, RC: &ARM::HPRRegClass, Op0, Op1);
5784 }
5785 return Register();
5786}
5787
5788Register fastEmit_ISD_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5789 if (RetVT.SimpleTy != MVT::f32)
5790 return Register();
5791 if ((Subtarget->hasFPARMv8Base())) {
5792 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMS, RC: &ARM::SPRRegClass, Op0, Op1);
5793 }
5794 return Register();
5795}
5796
5797Register fastEmit_ISD_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5798 if (RetVT.SimpleTy != MVT::f64)
5799 return Register();
5800 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
5801 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMD, RC: &ARM::DPRRegClass, Op0, Op1);
5802 }
5803 return Register();
5804}
5805
5806Register fastEmit_ISD_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5807 if (RetVT.SimpleTy != MVT::v4f16)
5808 return Register();
5809 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5810 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNDh, RC: &ARM::DPRRegClass, Op0, Op1);
5811 }
5812 return Register();
5813}
5814
5815Register fastEmit_ISD_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5816 if (RetVT.SimpleTy != MVT::v8f16)
5817 return Register();
5818 if ((Subtarget->hasMVEFloatOps())) {
5819 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINNMf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5820 }
5821 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5822 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNQh, RC: &ARM::QPRRegClass, Op0, Op1);
5823 }
5824 return Register();
5825}
5826
5827Register fastEmit_ISD_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5828 if (RetVT.SimpleTy != MVT::v2f32)
5829 return Register();
5830 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
5831 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNDf, RC: &ARM::DPRRegClass, Op0, Op1);
5832 }
5833 return Register();
5834}
5835
5836Register fastEmit_ISD_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5837 if (RetVT.SimpleTy != MVT::v4f32)
5838 return Register();
5839 if ((Subtarget->hasMVEFloatOps())) {
5840 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINNMf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5841 }
5842 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
5843 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNQf, RC: &ARM::QPRRegClass, Op0, Op1);
5844 }
5845 return Register();
5846}
5847
5848Register fastEmit_ISD_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5849 switch (VT.SimpleTy) {
5850 case MVT::f16: return fastEmit_ISD_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
5851 case MVT::f32: return fastEmit_ISD_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
5852 case MVT::f64: return fastEmit_ISD_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
5853 case MVT::v4f16: return fastEmit_ISD_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
5854 case MVT::v8f16: return fastEmit_ISD_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
5855 case MVT::v2f32: return fastEmit_ISD_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
5856 case MVT::v4f32: return fastEmit_ISD_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
5857 default: return Register();
5858 }
5859}
5860
5861// FastEmit functions for ISD::FMUL.
5862
5863Register fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5864 if (RetVT.SimpleTy != MVT::f16)
5865 return Register();
5866 if ((Subtarget->hasFullFP16())) {
5867 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULH, RC: &ARM::HPRRegClass, Op0, Op1);
5868 }
5869 return Register();
5870}
5871
5872Register fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5873 if (RetVT.SimpleTy != MVT::f32)
5874 return Register();
5875 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
5876 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULS, RC: &ARM::SPRRegClass, Op0, Op1);
5877 }
5878 return Register();
5879}
5880
5881Register fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5882 if (RetVT.SimpleTy != MVT::f64)
5883 return Register();
5884 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
5885 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULD, RC: &ARM::DPRRegClass, Op0, Op1);
5886 }
5887 return Register();
5888}
5889
5890Register fastEmit_ISD_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5891 if (RetVT.SimpleTy != MVT::v4f16)
5892 return Register();
5893 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5894 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULhd, RC: &ARM::DPRRegClass, Op0, Op1);
5895 }
5896 return Register();
5897}
5898
5899Register fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5900 if (RetVT.SimpleTy != MVT::v8f16)
5901 return Register();
5902 if ((Subtarget->hasMVEFloatOps())) {
5903 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5904 }
5905 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5906 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULhq, RC: &ARM::QPRRegClass, Op0, Op1);
5907 }
5908 return Register();
5909}
5910
5911Register fastEmit_ISD_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5912 if (RetVT.SimpleTy != MVT::v2f32)
5913 return Register();
5914 if ((Subtarget->hasNEON())) {
5915 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULfd, RC: &ARM::DPRRegClass, Op0, Op1);
5916 }
5917 return Register();
5918}
5919
5920Register fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5921 if (RetVT.SimpleTy != MVT::v4f32)
5922 return Register();
5923 if ((Subtarget->hasMVEFloatOps())) {
5924 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5925 }
5926 if ((Subtarget->hasNEON())) {
5927 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULfq, RC: &ARM::QPRRegClass, Op0, Op1);
5928 }
5929 return Register();
5930}
5931
5932Register fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5933 switch (VT.SimpleTy) {
5934 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
5935 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
5936 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
5937 case MVT::v4f16: return fastEmit_ISD_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
5938 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
5939 case MVT::v2f32: return fastEmit_ISD_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
5940 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
5941 default: return Register();
5942 }
5943}
5944
5945// FastEmit functions for ISD::FSUB.
5946
5947Register fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5948 if (RetVT.SimpleTy != MVT::f16)
5949 return Register();
5950 if ((Subtarget->hasFullFP16())) {
5951 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBH, RC: &ARM::HPRRegClass, Op0, Op1);
5952 }
5953 return Register();
5954}
5955
5956Register fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5957 if (RetVT.SimpleTy != MVT::f32)
5958 return Register();
5959 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
5960 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBS, RC: &ARM::SPRRegClass, Op0, Op1);
5961 }
5962 return Register();
5963}
5964
5965Register fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5966 if (RetVT.SimpleTy != MVT::f64)
5967 return Register();
5968 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
5969 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBD, RC: &ARM::DPRRegClass, Op0, Op1);
5970 }
5971 return Register();
5972}
5973
5974Register fastEmit_ISD_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5975 if (RetVT.SimpleTy != MVT::v4f16)
5976 return Register();
5977 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5978 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBhd, RC: &ARM::DPRRegClass, Op0, Op1);
5979 }
5980 return Register();
5981}
5982
5983Register fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5984 if (RetVT.SimpleTy != MVT::v8f16)
5985 return Register();
5986 if ((Subtarget->hasMVEFloatOps())) {
5987 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5988 }
5989 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5990 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBhq, RC: &ARM::QPRRegClass, Op0, Op1);
5991 }
5992 return Register();
5993}
5994
5995Register fastEmit_ISD_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5996 if (RetVT.SimpleTy != MVT::v2f32)
5997 return Register();
5998 if ((Subtarget->hasNEON())) {
5999 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBfd, RC: &ARM::DPRRegClass, Op0, Op1);
6000 }
6001 return Register();
6002}
6003
6004Register fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
6005 if (RetVT.SimpleTy != MVT::v4f32)
6006 return Register();
6007 if ((Subtarget->hasMVEFloatOps())) {
6008 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBf32, RC: &ARM::MQPRRegClass, Op0, Op1);
6009 }
6010 if ((Subtarget->hasNEON())) {
6011 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBfq, RC: &ARM::QPRRegClass, Op0, Op1);
6012 }
6013 return Register();
6014}
6015
6016Register fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6017 switch (VT.SimpleTy) {
6018 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
6019 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
6020 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
6021 case MVT::v4f16: return fastEmit_ISD_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
6022 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
6023 case MVT::v2f32: return fastEmit_ISD_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
6024 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
6025 default: return Register();
6026 }
6027}
6028
6029// FastEmit functions for ISD::MUL.
6030
6031Register fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6032 if (RetVT.SimpleTy != MVT::i32)
6033 return Register();
6034 if ((Subtarget->isThumb2())) {
6035 return fastEmitInst_rr(MachineInstOpcode: ARM::t2MUL, RC: &ARM::rGPRRegClass, Op0, Op1);
6036 }
6037 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6038 return fastEmitInst_rr(MachineInstOpcode: ARM::tMUL, RC: &ARM::tGPRRegClass, Op0, Op1);
6039 }
6040 if ((!Subtarget->isThumb()) && (!Subtarget->hasV6Ops()) && (Subtarget->useMulOps())) {
6041 return fastEmitInst_rr(MachineInstOpcode: ARM::MULv5, RC: &ARM::GPRnopcRegClass, Op0, Op1);
6042 }
6043 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
6044 return fastEmitInst_rr(MachineInstOpcode: ARM::MUL, RC: &ARM::GPRnopcRegClass, Op0, Op1);
6045 }
6046 return Register();
6047}
6048
6049Register fastEmit_ISD_MUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6050 if (RetVT.SimpleTy != MVT::v8i8)
6051 return Register();
6052 if ((Subtarget->hasNEON())) {
6053 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6054 }
6055 return Register();
6056}
6057
6058Register fastEmit_ISD_MUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6059 if (RetVT.SimpleTy != MVT::v16i8)
6060 return Register();
6061 if ((Subtarget->hasMVEIntegerOps())) {
6062 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULi8, RC: &ARM::MQPRRegClass, Op0, Op1);
6063 }
6064 if ((Subtarget->hasNEON())) {
6065 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6066 }
6067 return Register();
6068}
6069
6070Register fastEmit_ISD_MUL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6071 if (RetVT.SimpleTy != MVT::v4i16)
6072 return Register();
6073 if ((Subtarget->hasNEON())) {
6074 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6075 }
6076 return Register();
6077}
6078
6079Register fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6080 if (RetVT.SimpleTy != MVT::v8i16)
6081 return Register();
6082 if ((Subtarget->hasMVEIntegerOps())) {
6083 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULi16, RC: &ARM::MQPRRegClass, Op0, Op1);
6084 }
6085 if ((Subtarget->hasNEON())) {
6086 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6087 }
6088 return Register();
6089}
6090
6091Register fastEmit_ISD_MUL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6092 if (RetVT.SimpleTy != MVT::v2i32)
6093 return Register();
6094 if ((Subtarget->hasNEON())) {
6095 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6096 }
6097 return Register();
6098}
6099
6100Register fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6101 if (RetVT.SimpleTy != MVT::v4i32)
6102 return Register();
6103 if ((Subtarget->hasMVEIntegerOps())) {
6104 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULi32, RC: &ARM::MQPRRegClass, Op0, Op1);
6105 }
6106 if ((Subtarget->hasNEON())) {
6107 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6108 }
6109 return Register();
6110}
6111
6112Register fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6113 switch (VT.SimpleTy) {
6114 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op1);
6115 case MVT::v8i8: return fastEmit_ISD_MUL_MVT_v8i8_rr(RetVT, Op0, Op1);
6116 case MVT::v16i8: return fastEmit_ISD_MUL_MVT_v16i8_rr(RetVT, Op0, Op1);
6117 case MVT::v4i16: return fastEmit_ISD_MUL_MVT_v4i16_rr(RetVT, Op0, Op1);
6118 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
6119 case MVT::v2i32: return fastEmit_ISD_MUL_MVT_v2i32_rr(RetVT, Op0, Op1);
6120 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
6121 default: return Register();
6122 }
6123}
6124
6125// FastEmit functions for ISD::MULHS.
6126
6127Register fastEmit_ISD_MULHS_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6128 if (RetVT.SimpleTy != MVT::i32)
6129 return Register();
6130 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
6131 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SMMUL, RC: &ARM::rGPRRegClass, Op0, Op1);
6132 }
6133 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
6134 return fastEmitInst_rr(MachineInstOpcode: ARM::SMMUL, RC: &ARM::GPRRegClass, Op0, Op1);
6135 }
6136 return Register();
6137}
6138
6139Register fastEmit_ISD_MULHS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6140 if (RetVT.SimpleTy != MVT::v16i8)
6141 return Register();
6142 if ((Subtarget->hasMVEIntegerOps())) {
6143 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHs8, RC: &ARM::MQPRRegClass, Op0, Op1);
6144 }
6145 return Register();
6146}
6147
6148Register fastEmit_ISD_MULHS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6149 if (RetVT.SimpleTy != MVT::v8i16)
6150 return Register();
6151 if ((Subtarget->hasMVEIntegerOps())) {
6152 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHs16, RC: &ARM::MQPRRegClass, Op0, Op1);
6153 }
6154 return Register();
6155}
6156
6157Register fastEmit_ISD_MULHS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6158 if (RetVT.SimpleTy != MVT::v4i32)
6159 return Register();
6160 if ((Subtarget->hasMVEIntegerOps())) {
6161 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHs32, RC: &ARM::MQPRRegClass, Op0, Op1);
6162 }
6163 return Register();
6164}
6165
6166Register fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6167 switch (VT.SimpleTy) {
6168 case MVT::i32: return fastEmit_ISD_MULHS_MVT_i32_rr(RetVT, Op0, Op1);
6169 case MVT::v16i8: return fastEmit_ISD_MULHS_MVT_v16i8_rr(RetVT, Op0, Op1);
6170 case MVT::v8i16: return fastEmit_ISD_MULHS_MVT_v8i16_rr(RetVT, Op0, Op1);
6171 case MVT::v4i32: return fastEmit_ISD_MULHS_MVT_v4i32_rr(RetVT, Op0, Op1);
6172 default: return Register();
6173 }
6174}
6175
6176// FastEmit functions for ISD::MULHU.
6177
6178Register fastEmit_ISD_MULHU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6179 if (RetVT.SimpleTy != MVT::v16i8)
6180 return Register();
6181 if ((Subtarget->hasMVEIntegerOps())) {
6182 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHu8, RC: &ARM::MQPRRegClass, Op0, Op1);
6183 }
6184 return Register();
6185}
6186
6187Register fastEmit_ISD_MULHU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6188 if (RetVT.SimpleTy != MVT::v8i16)
6189 return Register();
6190 if ((Subtarget->hasMVEIntegerOps())) {
6191 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHu16, RC: &ARM::MQPRRegClass, Op0, Op1);
6192 }
6193 return Register();
6194}
6195
6196Register fastEmit_ISD_MULHU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6197 if (RetVT.SimpleTy != MVT::v4i32)
6198 return Register();
6199 if ((Subtarget->hasMVEIntegerOps())) {
6200 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHu32, RC: &ARM::MQPRRegClass, Op0, Op1);
6201 }
6202 return Register();
6203}
6204
6205Register fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6206 switch (VT.SimpleTy) {
6207 case MVT::v16i8: return fastEmit_ISD_MULHU_MVT_v16i8_rr(RetVT, Op0, Op1);
6208 case MVT::v8i16: return fastEmit_ISD_MULHU_MVT_v8i16_rr(RetVT, Op0, Op1);
6209 case MVT::v4i32: return fastEmit_ISD_MULHU_MVT_v4i32_rr(RetVT, Op0, Op1);
6210 default: return Register();
6211 }
6212}
6213
6214// FastEmit functions for ISD::OR.
6215
6216Register fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6217 if (RetVT.SimpleTy != MVT::i32)
6218 return Register();
6219 if ((Subtarget->isThumb2())) {
6220 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ORRrr, RC: &ARM::rGPRRegClass, Op0, Op1);
6221 }
6222 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6223 return fastEmitInst_rr(MachineInstOpcode: ARM::tORR, RC: &ARM::tGPRRegClass, Op0, Op1);
6224 }
6225 if ((!Subtarget->isThumb())) {
6226 return fastEmitInst_rr(MachineInstOpcode: ARM::ORRrr, RC: &ARM::GPRRegClass, Op0, Op1);
6227 }
6228 return Register();
6229}
6230
6231Register fastEmit_ISD_OR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6232 if (RetVT.SimpleTy != MVT::v8i8)
6233 return Register();
6234 if ((Subtarget->hasNEON())) {
6235 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
6236 }
6237 return Register();
6238}
6239
6240Register fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6241 if (RetVT.SimpleTy != MVT::v16i8)
6242 return Register();
6243 if ((Subtarget->hasMVEIntegerOps())) {
6244 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
6245 }
6246 if ((Subtarget->hasNEON())) {
6247 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
6248 }
6249 return Register();
6250}
6251
6252Register fastEmit_ISD_OR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6253 if (RetVT.SimpleTy != MVT::v4i16)
6254 return Register();
6255 if ((Subtarget->hasNEON())) {
6256 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
6257 }
6258 return Register();
6259}
6260
6261Register fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6262 if (RetVT.SimpleTy != MVT::v8i16)
6263 return Register();
6264 if ((Subtarget->hasMVEIntegerOps())) {
6265 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
6266 }
6267 if ((Subtarget->hasNEON())) {
6268 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
6269 }
6270 return Register();
6271}
6272
6273Register fastEmit_ISD_OR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6274 if (RetVT.SimpleTy != MVT::v2i32)
6275 return Register();
6276 if ((Subtarget->hasNEON())) {
6277 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
6278 }
6279 return Register();
6280}
6281
6282Register fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6283 if (RetVT.SimpleTy != MVT::v4i32)
6284 return Register();
6285 if ((Subtarget->hasMVEIntegerOps())) {
6286 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
6287 }
6288 if ((Subtarget->hasNEON())) {
6289 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
6290 }
6291 return Register();
6292}
6293
6294Register fastEmit_ISD_OR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
6295 if (RetVT.SimpleTy != MVT::v1i64)
6296 return Register();
6297 if ((Subtarget->hasNEON())) {
6298 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
6299 }
6300 return Register();
6301}
6302
6303Register fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6304 if (RetVT.SimpleTy != MVT::v2i64)
6305 return Register();
6306 if ((Subtarget->hasMVEIntegerOps())) {
6307 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
6308 }
6309 if ((Subtarget->hasNEON())) {
6310 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
6311 }
6312 return Register();
6313}
6314
6315Register fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6316 switch (VT.SimpleTy) {
6317 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
6318 case MVT::v8i8: return fastEmit_ISD_OR_MVT_v8i8_rr(RetVT, Op0, Op1);
6319 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
6320 case MVT::v4i16: return fastEmit_ISD_OR_MVT_v4i16_rr(RetVT, Op0, Op1);
6321 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
6322 case MVT::v2i32: return fastEmit_ISD_OR_MVT_v2i32_rr(RetVT, Op0, Op1);
6323 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
6324 case MVT::v1i64: return fastEmit_ISD_OR_MVT_v1i64_rr(RetVT, Op0, Op1);
6325 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
6326 default: return Register();
6327 }
6328}
6329
6330// FastEmit functions for ISD::ROTR.
6331
6332Register fastEmit_ISD_ROTR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6333 if (RetVT.SimpleTy != MVT::i32)
6334 return Register();
6335 if ((Subtarget->isThumb2())) {
6336 return fastEmitInst_rr(MachineInstOpcode: ARM::t2RORrr, RC: &ARM::rGPRRegClass, Op0, Op1);
6337 }
6338 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6339 return fastEmitInst_rr(MachineInstOpcode: ARM::tROR, RC: &ARM::tGPRRegClass, Op0, Op1);
6340 }
6341 return Register();
6342}
6343
6344Register fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6345 switch (VT.SimpleTy) {
6346 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_rr(RetVT, Op0, Op1);
6347 default: return Register();
6348 }
6349}
6350
6351// FastEmit functions for ISD::SADDSAT.
6352
6353Register fastEmit_ISD_SADDSAT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6354 if (RetVT.SimpleTy != MVT::i32)
6355 return Register();
6356 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
6357 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QADD, RC: &ARM::rGPRRegClass, Op0, Op1);
6358 }
6359 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
6360 return fastEmitInst_rr(MachineInstOpcode: ARM::QADD, RC: &ARM::GPRnopcRegClass, Op0, Op1);
6361 }
6362 return Register();
6363}
6364
6365Register fastEmit_ISD_SADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6366 if (RetVT.SimpleTy != MVT::v8i8)
6367 return Register();
6368 if ((Subtarget->hasNEON())) {
6369 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6370 }
6371 return Register();
6372}
6373
6374Register fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6375 if (RetVT.SimpleTy != MVT::v16i8)
6376 return Register();
6377 if ((Subtarget->hasMVEIntegerOps())) {
6378 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
6379 }
6380 if ((Subtarget->hasNEON())) {
6381 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6382 }
6383 return Register();
6384}
6385
6386Register fastEmit_ISD_SADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6387 if (RetVT.SimpleTy != MVT::v4i16)
6388 return Register();
6389 if ((Subtarget->hasNEON())) {
6390 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6391 }
6392 return Register();
6393}
6394
6395Register fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6396 if (RetVT.SimpleTy != MVT::v8i16)
6397 return Register();
6398 if ((Subtarget->hasMVEIntegerOps())) {
6399 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
6400 }
6401 if ((Subtarget->hasNEON())) {
6402 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6403 }
6404 return Register();
6405}
6406
6407Register fastEmit_ISD_SADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6408 if (RetVT.SimpleTy != MVT::v2i32)
6409 return Register();
6410 if ((Subtarget->hasNEON())) {
6411 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6412 }
6413 return Register();
6414}
6415
6416Register fastEmit_ISD_SADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6417 if (RetVT.SimpleTy != MVT::v4i32)
6418 return Register();
6419 if ((Subtarget->hasMVEIntegerOps())) {
6420 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
6421 }
6422 if ((Subtarget->hasNEON())) {
6423 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6424 }
6425 return Register();
6426}
6427
6428Register fastEmit_ISD_SADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
6429 if (RetVT.SimpleTy != MVT::v1i64)
6430 return Register();
6431 if ((Subtarget->hasNEON())) {
6432 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
6433 }
6434 return Register();
6435}
6436
6437Register fastEmit_ISD_SADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6438 if (RetVT.SimpleTy != MVT::v2i64)
6439 return Register();
6440 if ((Subtarget->hasNEON())) {
6441 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
6442 }
6443 return Register();
6444}
6445
6446Register fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6447 switch (VT.SimpleTy) {
6448 case MVT::i32: return fastEmit_ISD_SADDSAT_MVT_i32_rr(RetVT, Op0, Op1);
6449 case MVT::v8i8: return fastEmit_ISD_SADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
6450 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
6451 case MVT::v4i16: return fastEmit_ISD_SADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
6452 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
6453 case MVT::v2i32: return fastEmit_ISD_SADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
6454 case MVT::v4i32: return fastEmit_ISD_SADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
6455 case MVT::v1i64: return fastEmit_ISD_SADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
6456 case MVT::v2i64: return fastEmit_ISD_SADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
6457 default: return Register();
6458 }
6459}
6460
6461// FastEmit functions for ISD::SDIV.
6462
6463Register fastEmit_ISD_SDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6464 if (RetVT.SimpleTy != MVT::i32)
6465 return Register();
6466 if ((Subtarget->hasDivideInThumbMode()) && (Subtarget->hasV8MBaselineOps()) && (Subtarget->isThumb())) {
6467 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SDIV, RC: &ARM::rGPRRegClass, Op0, Op1);
6468 }
6469 if ((Subtarget->hasDivideInARMMode()) && (!Subtarget->isThumb())) {
6470 return fastEmitInst_rr(MachineInstOpcode: ARM::SDIV, RC: &ARM::GPRRegClass, Op0, Op1);
6471 }
6472 return Register();
6473}
6474
6475Register fastEmit_ISD_SDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6476 switch (VT.SimpleTy) {
6477 case MVT::i32: return fastEmit_ISD_SDIV_MVT_i32_rr(RetVT, Op0, Op1);
6478 default: return Register();
6479 }
6480}
6481
6482// FastEmit functions for ISD::SHL.
6483
6484Register fastEmit_ISD_SHL_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6485 if (RetVT.SimpleTy != MVT::i32)
6486 return Register();
6487 if ((Subtarget->isThumb2())) {
6488 return fastEmitInst_rr(MachineInstOpcode: ARM::t2LSLrr, RC: &ARM::rGPRRegClass, Op0, Op1);
6489 }
6490 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6491 return fastEmitInst_rr(MachineInstOpcode: ARM::tLSLrr, RC: &ARM::tGPRRegClass, Op0, Op1);
6492 }
6493 return Register();
6494}
6495
6496Register fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6497 switch (VT.SimpleTy) {
6498 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_rr(RetVT, Op0, Op1);
6499 default: return Register();
6500 }
6501}
6502
6503// FastEmit functions for ISD::SMAX.
6504
6505Register fastEmit_ISD_SMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6506 if (RetVT.SimpleTy != MVT::v8i8)
6507 return Register();
6508 if ((Subtarget->hasNEON())) {
6509 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6510 }
6511 return Register();
6512}
6513
6514Register fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6515 if (RetVT.SimpleTy != MVT::v16i8)
6516 return Register();
6517 if ((Subtarget->hasMVEIntegerOps())) {
6518 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXs8, RC: &ARM::MQPRRegClass, Op0, Op1);
6519 }
6520 if ((Subtarget->hasNEON())) {
6521 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6522 }
6523 return Register();
6524}
6525
6526Register fastEmit_ISD_SMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6527 if (RetVT.SimpleTy != MVT::v4i16)
6528 return Register();
6529 if ((Subtarget->hasNEON())) {
6530 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6531 }
6532 return Register();
6533}
6534
6535Register fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6536 if (RetVT.SimpleTy != MVT::v8i16)
6537 return Register();
6538 if ((Subtarget->hasMVEIntegerOps())) {
6539 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXs16, RC: &ARM::MQPRRegClass, Op0, Op1);
6540 }
6541 if ((Subtarget->hasNEON())) {
6542 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6543 }
6544 return Register();
6545}
6546
6547Register fastEmit_ISD_SMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6548 if (RetVT.SimpleTy != MVT::v2i32)
6549 return Register();
6550 if ((Subtarget->hasNEON())) {
6551 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6552 }
6553 return Register();
6554}
6555
6556Register fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6557 if (RetVT.SimpleTy != MVT::v4i32)
6558 return Register();
6559 if ((Subtarget->hasMVEIntegerOps())) {
6560 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXs32, RC: &ARM::MQPRRegClass, Op0, Op1);
6561 }
6562 if ((Subtarget->hasNEON())) {
6563 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6564 }
6565 return Register();
6566}
6567
6568Register fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6569 switch (VT.SimpleTy) {
6570 case MVT::v8i8: return fastEmit_ISD_SMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
6571 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
6572 case MVT::v4i16: return fastEmit_ISD_SMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
6573 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
6574 case MVT::v2i32: return fastEmit_ISD_SMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
6575 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
6576 default: return Register();
6577 }
6578}
6579
6580// FastEmit functions for ISD::SMIN.
6581
6582Register fastEmit_ISD_SMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6583 if (RetVT.SimpleTy != MVT::v8i8)
6584 return Register();
6585 if ((Subtarget->hasNEON())) {
6586 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6587 }
6588 return Register();
6589}
6590
6591Register fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6592 if (RetVT.SimpleTy != MVT::v16i8)
6593 return Register();
6594 if ((Subtarget->hasMVEIntegerOps())) {
6595 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINs8, RC: &ARM::MQPRRegClass, Op0, Op1);
6596 }
6597 if ((Subtarget->hasNEON())) {
6598 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6599 }
6600 return Register();
6601}
6602
6603Register fastEmit_ISD_SMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6604 if (RetVT.SimpleTy != MVT::v4i16)
6605 return Register();
6606 if ((Subtarget->hasNEON())) {
6607 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6608 }
6609 return Register();
6610}
6611
6612Register fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6613 if (RetVT.SimpleTy != MVT::v8i16)
6614 return Register();
6615 if ((Subtarget->hasMVEIntegerOps())) {
6616 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINs16, RC: &ARM::MQPRRegClass, Op0, Op1);
6617 }
6618 if ((Subtarget->hasNEON())) {
6619 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6620 }
6621 return Register();
6622}
6623
6624Register fastEmit_ISD_SMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6625 if (RetVT.SimpleTy != MVT::v2i32)
6626 return Register();
6627 if ((Subtarget->hasNEON())) {
6628 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6629 }
6630 return Register();
6631}
6632
6633Register fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6634 if (RetVT.SimpleTy != MVT::v4i32)
6635 return Register();
6636 if ((Subtarget->hasMVEIntegerOps())) {
6637 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINs32, RC: &ARM::MQPRRegClass, Op0, Op1);
6638 }
6639 if ((Subtarget->hasNEON())) {
6640 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6641 }
6642 return Register();
6643}
6644
6645Register fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6646 switch (VT.SimpleTy) {
6647 case MVT::v8i8: return fastEmit_ISD_SMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
6648 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
6649 case MVT::v4i16: return fastEmit_ISD_SMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
6650 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
6651 case MVT::v2i32: return fastEmit_ISD_SMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
6652 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
6653 default: return Register();
6654 }
6655}
6656
6657// FastEmit functions for ISD::SRA.
6658
6659Register fastEmit_ISD_SRA_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6660 if (RetVT.SimpleTy != MVT::i32)
6661 return Register();
6662 if ((Subtarget->isThumb2())) {
6663 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ASRrr, RC: &ARM::rGPRRegClass, Op0, Op1);
6664 }
6665 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6666 return fastEmitInst_rr(MachineInstOpcode: ARM::tASRrr, RC: &ARM::tGPRRegClass, Op0, Op1);
6667 }
6668 return Register();
6669}
6670
6671Register fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6672 switch (VT.SimpleTy) {
6673 case MVT::i32: return fastEmit_ISD_SRA_MVT_i32_rr(RetVT, Op0, Op1);
6674 default: return Register();
6675 }
6676}
6677
6678// FastEmit functions for ISD::SRL.
6679
6680Register fastEmit_ISD_SRL_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6681 if (RetVT.SimpleTy != MVT::i32)
6682 return Register();
6683 if ((Subtarget->isThumb2())) {
6684 return fastEmitInst_rr(MachineInstOpcode: ARM::t2LSRrr, RC: &ARM::rGPRRegClass, Op0, Op1);
6685 }
6686 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6687 return fastEmitInst_rr(MachineInstOpcode: ARM::tLSRrr, RC: &ARM::tGPRRegClass, Op0, Op1);
6688 }
6689 return Register();
6690}
6691
6692Register fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6693 switch (VT.SimpleTy) {
6694 case MVT::i32: return fastEmit_ISD_SRL_MVT_i32_rr(RetVT, Op0, Op1);
6695 default: return Register();
6696 }
6697}
6698
6699// FastEmit functions for ISD::SSUBSAT.
6700
6701Register fastEmit_ISD_SSUBSAT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6702 if (RetVT.SimpleTy != MVT::i32)
6703 return Register();
6704 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
6705 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QSUB, RC: &ARM::rGPRRegClass, Op0, Op1);
6706 }
6707 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
6708 return fastEmitInst_rr(MachineInstOpcode: ARM::QSUB, RC: &ARM::GPRnopcRegClass, Op0, Op1);
6709 }
6710 return Register();
6711}
6712
6713Register fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6714 if (RetVT.SimpleTy != MVT::v8i8)
6715 return Register();
6716 if ((Subtarget->hasNEON())) {
6717 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6718 }
6719 return Register();
6720}
6721
6722Register fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6723 if (RetVT.SimpleTy != MVT::v16i8)
6724 return Register();
6725 if ((Subtarget->hasMVEIntegerOps())) {
6726 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBs8, RC: &ARM::MQPRRegClass, Op0, Op1);
6727 }
6728 if ((Subtarget->hasNEON())) {
6729 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6730 }
6731 return Register();
6732}
6733
6734Register fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6735 if (RetVT.SimpleTy != MVT::v4i16)
6736 return Register();
6737 if ((Subtarget->hasNEON())) {
6738 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6739 }
6740 return Register();
6741}
6742
6743Register fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6744 if (RetVT.SimpleTy != MVT::v8i16)
6745 return Register();
6746 if ((Subtarget->hasMVEIntegerOps())) {
6747 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBs16, RC: &ARM::MQPRRegClass, Op0, Op1);
6748 }
6749 if ((Subtarget->hasNEON())) {
6750 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6751 }
6752 return Register();
6753}
6754
6755Register fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6756 if (RetVT.SimpleTy != MVT::v2i32)
6757 return Register();
6758 if ((Subtarget->hasNEON())) {
6759 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6760 }
6761 return Register();
6762}
6763
6764Register fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6765 if (RetVT.SimpleTy != MVT::v4i32)
6766 return Register();
6767 if ((Subtarget->hasMVEIntegerOps())) {
6768 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBs32, RC: &ARM::MQPRRegClass, Op0, Op1);
6769 }
6770 if ((Subtarget->hasNEON())) {
6771 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6772 }
6773 return Register();
6774}
6775
6776Register fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
6777 if (RetVT.SimpleTy != MVT::v1i64)
6778 return Register();
6779 if ((Subtarget->hasNEON())) {
6780 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
6781 }
6782 return Register();
6783}
6784
6785Register fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6786 if (RetVT.SimpleTy != MVT::v2i64)
6787 return Register();
6788 if ((Subtarget->hasNEON())) {
6789 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
6790 }
6791 return Register();
6792}
6793
6794Register fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6795 switch (VT.SimpleTy) {
6796 case MVT::i32: return fastEmit_ISD_SSUBSAT_MVT_i32_rr(RetVT, Op0, Op1);
6797 case MVT::v8i8: return fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
6798 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
6799 case MVT::v4i16: return fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
6800 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
6801 case MVT::v2i32: return fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
6802 case MVT::v4i32: return fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
6803 case MVT::v1i64: return fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
6804 case MVT::v2i64: return fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
6805 default: return Register();
6806 }
6807}
6808
6809// FastEmit functions for ISD::STRICT_FADD.
6810
6811Register fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6812 if (RetVT.SimpleTy != MVT::f16)
6813 return Register();
6814 if ((Subtarget->hasFullFP16())) {
6815 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDH, RC: &ARM::HPRRegClass, Op0, Op1);
6816 }
6817 return Register();
6818}
6819
6820Register fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6821 if (RetVT.SimpleTy != MVT::f32)
6822 return Register();
6823 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
6824 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDS, RC: &ARM::SPRRegClass, Op0, Op1);
6825 }
6826 return Register();
6827}
6828
6829Register fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6830 if (RetVT.SimpleTy != MVT::f64)
6831 return Register();
6832 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
6833 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDD, RC: &ARM::DPRRegClass, Op0, Op1);
6834 }
6835 return Register();
6836}
6837
6838Register fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6839 switch (VT.SimpleTy) {
6840 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1);
6841 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1);
6842 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1);
6843 default: return Register();
6844 }
6845}
6846
6847// FastEmit functions for ISD::STRICT_FDIV.
6848
6849Register fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6850 if (RetVT.SimpleTy != MVT::f16)
6851 return Register();
6852 if ((Subtarget->hasFullFP16())) {
6853 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVH, RC: &ARM::HPRRegClass, Op0, Op1);
6854 }
6855 return Register();
6856}
6857
6858Register fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6859 if (RetVT.SimpleTy != MVT::f32)
6860 return Register();
6861 if ((Subtarget->hasVFP2Base())) {
6862 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVS, RC: &ARM::SPRRegClass, Op0, Op1);
6863 }
6864 return Register();
6865}
6866
6867Register fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6868 if (RetVT.SimpleTy != MVT::f64)
6869 return Register();
6870 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
6871 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVD, RC: &ARM::DPRRegClass, Op0, Op1);
6872 }
6873 return Register();
6874}
6875
6876Register fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6877 switch (VT.SimpleTy) {
6878 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
6879 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
6880 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
6881 default: return Register();
6882 }
6883}
6884
6885// FastEmit functions for ISD::STRICT_FMAXNUM.
6886
6887Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6888 if (RetVT.SimpleTy != MVT::f16)
6889 return Register();
6890 if ((Subtarget->hasFullFP16())) {
6891 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMH, RC: &ARM::HPRRegClass, Op0, Op1);
6892 }
6893 return Register();
6894}
6895
6896Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6897 if (RetVT.SimpleTy != MVT::f32)
6898 return Register();
6899 if ((Subtarget->hasFPARMv8Base())) {
6900 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMS, RC: &ARM::SPRRegClass, Op0, Op1);
6901 }
6902 return Register();
6903}
6904
6905Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6906 if (RetVT.SimpleTy != MVT::f64)
6907 return Register();
6908 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
6909 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMD, RC: &ARM::DPRRegClass, Op0, Op1);
6910 }
6911 return Register();
6912}
6913
6914Register fastEmit_ISD_STRICT_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6915 switch (VT.SimpleTy) {
6916 case MVT::f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
6917 case MVT::f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
6918 case MVT::f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
6919 default: return Register();
6920 }
6921}
6922
6923// FastEmit functions for ISD::STRICT_FMINNUM.
6924
6925Register fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6926 if (RetVT.SimpleTy != MVT::f16)
6927 return Register();
6928 if ((Subtarget->hasFullFP16())) {
6929 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMH, RC: &ARM::HPRRegClass, Op0, Op1);
6930 }
6931 return Register();
6932}
6933
6934Register fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6935 if (RetVT.SimpleTy != MVT::f32)
6936 return Register();
6937 if ((Subtarget->hasFPARMv8Base())) {
6938 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMS, RC: &ARM::SPRRegClass, Op0, Op1);
6939 }
6940 return Register();
6941}
6942
6943Register fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6944 if (RetVT.SimpleTy != MVT::f64)
6945 return Register();
6946 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
6947 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMD, RC: &ARM::DPRRegClass, Op0, Op1);
6948 }
6949 return Register();
6950}
6951
6952Register fastEmit_ISD_STRICT_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6953 switch (VT.SimpleTy) {
6954 case MVT::f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
6955 case MVT::f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
6956 case MVT::f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
6957 default: return Register();
6958 }
6959}
6960
6961// FastEmit functions for ISD::STRICT_FMUL.
6962
6963Register fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6964 if (RetVT.SimpleTy != MVT::f16)
6965 return Register();
6966 if ((Subtarget->hasFullFP16())) {
6967 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULH, RC: &ARM::HPRRegClass, Op0, Op1);
6968 }
6969 return Register();
6970}
6971
6972Register fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6973 if (RetVT.SimpleTy != MVT::f32)
6974 return Register();
6975 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
6976 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULS, RC: &ARM::SPRRegClass, Op0, Op1);
6977 }
6978 return Register();
6979}
6980
6981Register fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6982 if (RetVT.SimpleTy != MVT::f64)
6983 return Register();
6984 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
6985 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULD, RC: &ARM::DPRRegClass, Op0, Op1);
6986 }
6987 return Register();
6988}
6989
6990Register fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6991 switch (VT.SimpleTy) {
6992 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
6993 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
6994 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
6995 default: return Register();
6996 }
6997}
6998
6999// FastEmit functions for ISD::STRICT_FSUB.
7000
7001Register fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
7002 if (RetVT.SimpleTy != MVT::f16)
7003 return Register();
7004 if ((Subtarget->hasFullFP16())) {
7005 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBH, RC: &ARM::HPRRegClass, Op0, Op1);
7006 }
7007 return Register();
7008}
7009
7010Register fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7011 if (RetVT.SimpleTy != MVT::f32)
7012 return Register();
7013 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
7014 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBS, RC: &ARM::SPRRegClass, Op0, Op1);
7015 }
7016 return Register();
7017}
7018
7019Register fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7020 if (RetVT.SimpleTy != MVT::f64)
7021 return Register();
7022 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
7023 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBD, RC: &ARM::DPRRegClass, Op0, Op1);
7024 }
7025 return Register();
7026}
7027
7028Register fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7029 switch (VT.SimpleTy) {
7030 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
7031 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
7032 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
7033 default: return Register();
7034 }
7035}
7036
7037// FastEmit functions for ISD::SUB.
7038
7039Register fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
7040 if (RetVT.SimpleTy != MVT::i32)
7041 return Register();
7042 if ((Subtarget->isThumb2())) {
7043 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SUBrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
7044 }
7045 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
7046 return fastEmitInst_rr(MachineInstOpcode: ARM::tSUBrr, RC: &ARM::tGPRRegClass, Op0, Op1);
7047 }
7048 if ((!Subtarget->isThumb())) {
7049 return fastEmitInst_rr(MachineInstOpcode: ARM::SUBrr, RC: &ARM::GPRRegClass, Op0, Op1);
7050 }
7051 return Register();
7052}
7053
7054Register fastEmit_ISD_SUB_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7055 if (RetVT.SimpleTy != MVT::v8i8)
7056 return Register();
7057 if ((Subtarget->hasNEON())) {
7058 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
7059 }
7060 return Register();
7061}
7062
7063Register fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7064 if (RetVT.SimpleTy != MVT::v16i8)
7065 return Register();
7066 if ((Subtarget->hasMVEIntegerOps())) {
7067 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBi8, RC: &ARM::MQPRRegClass, Op0, Op1);
7068 }
7069 if ((Subtarget->hasNEON())) {
7070 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
7071 }
7072 return Register();
7073}
7074
7075Register fastEmit_ISD_SUB_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7076 if (RetVT.SimpleTy != MVT::v4i16)
7077 return Register();
7078 if ((Subtarget->hasNEON())) {
7079 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
7080 }
7081 return Register();
7082}
7083
7084Register fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7085 if (RetVT.SimpleTy != MVT::v8i16)
7086 return Register();
7087 if ((Subtarget->hasMVEIntegerOps())) {
7088 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBi16, RC: &ARM::MQPRRegClass, Op0, Op1);
7089 }
7090 if ((Subtarget->hasNEON())) {
7091 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
7092 }
7093 return Register();
7094}
7095
7096Register fastEmit_ISD_SUB_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7097 if (RetVT.SimpleTy != MVT::v2i32)
7098 return Register();
7099 if ((Subtarget->hasNEON())) {
7100 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
7101 }
7102 return Register();
7103}
7104
7105Register fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7106 if (RetVT.SimpleTy != MVT::v4i32)
7107 return Register();
7108 if ((Subtarget->hasMVEIntegerOps())) {
7109 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBi32, RC: &ARM::MQPRRegClass, Op0, Op1);
7110 }
7111 if ((Subtarget->hasNEON())) {
7112 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
7113 }
7114 return Register();
7115}
7116
7117Register fastEmit_ISD_SUB_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7118 if (RetVT.SimpleTy != MVT::v1i64)
7119 return Register();
7120 if ((Subtarget->hasNEON())) {
7121 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
7122 }
7123 return Register();
7124}
7125
7126Register fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7127 if (RetVT.SimpleTy != MVT::v2i64)
7128 return Register();
7129 if ((Subtarget->hasNEON())) {
7130 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
7131 }
7132 return Register();
7133}
7134
7135Register fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7136 switch (VT.SimpleTy) {
7137 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
7138 case MVT::v8i8: return fastEmit_ISD_SUB_MVT_v8i8_rr(RetVT, Op0, Op1);
7139 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
7140 case MVT::v4i16: return fastEmit_ISD_SUB_MVT_v4i16_rr(RetVT, Op0, Op1);
7141 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
7142 case MVT::v2i32: return fastEmit_ISD_SUB_MVT_v2i32_rr(RetVT, Op0, Op1);
7143 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
7144 case MVT::v1i64: return fastEmit_ISD_SUB_MVT_v1i64_rr(RetVT, Op0, Op1);
7145 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
7146 default: return Register();
7147 }
7148}
7149
7150// FastEmit functions for ISD::UADDSAT.
7151
7152Register fastEmit_ISD_UADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7153 if (RetVT.SimpleTy != MVT::v8i8)
7154 return Register();
7155 if ((Subtarget->hasNEON())) {
7156 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
7157 }
7158 return Register();
7159}
7160
7161Register fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7162 if (RetVT.SimpleTy != MVT::v16i8)
7163 return Register();
7164 if ((Subtarget->hasMVEIntegerOps())) {
7165 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
7166 }
7167 if ((Subtarget->hasNEON())) {
7168 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
7169 }
7170 return Register();
7171}
7172
7173Register fastEmit_ISD_UADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7174 if (RetVT.SimpleTy != MVT::v4i16)
7175 return Register();
7176 if ((Subtarget->hasNEON())) {
7177 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
7178 }
7179 return Register();
7180}
7181
7182Register fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7183 if (RetVT.SimpleTy != MVT::v8i16)
7184 return Register();
7185 if ((Subtarget->hasMVEIntegerOps())) {
7186 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
7187 }
7188 if ((Subtarget->hasNEON())) {
7189 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
7190 }
7191 return Register();
7192}
7193
7194Register fastEmit_ISD_UADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7195 if (RetVT.SimpleTy != MVT::v2i32)
7196 return Register();
7197 if ((Subtarget->hasNEON())) {
7198 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
7199 }
7200 return Register();
7201}
7202
7203Register fastEmit_ISD_UADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7204 if (RetVT.SimpleTy != MVT::v4i32)
7205 return Register();
7206 if ((Subtarget->hasMVEIntegerOps())) {
7207 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
7208 }
7209 if ((Subtarget->hasNEON())) {
7210 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
7211 }
7212 return Register();
7213}
7214
7215Register fastEmit_ISD_UADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7216 if (RetVT.SimpleTy != MVT::v1i64)
7217 return Register();
7218 if ((Subtarget->hasNEON())) {
7219 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
7220 }
7221 return Register();
7222}
7223
7224Register fastEmit_ISD_UADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7225 if (RetVT.SimpleTy != MVT::v2i64)
7226 return Register();
7227 if ((Subtarget->hasNEON())) {
7228 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
7229 }
7230 return Register();
7231}
7232
7233Register fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7234 switch (VT.SimpleTy) {
7235 case MVT::v8i8: return fastEmit_ISD_UADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
7236 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
7237 case MVT::v4i16: return fastEmit_ISD_UADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
7238 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
7239 case MVT::v2i32: return fastEmit_ISD_UADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
7240 case MVT::v4i32: return fastEmit_ISD_UADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
7241 case MVT::v1i64: return fastEmit_ISD_UADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
7242 case MVT::v2i64: return fastEmit_ISD_UADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
7243 default: return Register();
7244 }
7245}
7246
7247// FastEmit functions for ISD::UDIV.
7248
7249Register fastEmit_ISD_UDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
7250 if (RetVT.SimpleTy != MVT::i32)
7251 return Register();
7252 if ((Subtarget->hasDivideInThumbMode()) && (Subtarget->hasV8MBaselineOps()) && (Subtarget->isThumb())) {
7253 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UDIV, RC: &ARM::rGPRRegClass, Op0, Op1);
7254 }
7255 if ((Subtarget->hasDivideInARMMode()) && (!Subtarget->isThumb())) {
7256 return fastEmitInst_rr(MachineInstOpcode: ARM::UDIV, RC: &ARM::GPRRegClass, Op0, Op1);
7257 }
7258 return Register();
7259}
7260
7261Register fastEmit_ISD_UDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7262 switch (VT.SimpleTy) {
7263 case MVT::i32: return fastEmit_ISD_UDIV_MVT_i32_rr(RetVT, Op0, Op1);
7264 default: return Register();
7265 }
7266}
7267
7268// FastEmit functions for ISD::UMAX.
7269
7270Register fastEmit_ISD_UMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7271 if (RetVT.SimpleTy != MVT::v8i8)
7272 return Register();
7273 if ((Subtarget->hasNEON())) {
7274 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
7275 }
7276 return Register();
7277}
7278
7279Register fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7280 if (RetVT.SimpleTy != MVT::v16i8)
7281 return Register();
7282 if ((Subtarget->hasMVEIntegerOps())) {
7283 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXu8, RC: &ARM::MQPRRegClass, Op0, Op1);
7284 }
7285 if ((Subtarget->hasNEON())) {
7286 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
7287 }
7288 return Register();
7289}
7290
7291Register fastEmit_ISD_UMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7292 if (RetVT.SimpleTy != MVT::v4i16)
7293 return Register();
7294 if ((Subtarget->hasNEON())) {
7295 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
7296 }
7297 return Register();
7298}
7299
7300Register fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7301 if (RetVT.SimpleTy != MVT::v8i16)
7302 return Register();
7303 if ((Subtarget->hasMVEIntegerOps())) {
7304 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXu16, RC: &ARM::MQPRRegClass, Op0, Op1);
7305 }
7306 if ((Subtarget->hasNEON())) {
7307 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
7308 }
7309 return Register();
7310}
7311
7312Register fastEmit_ISD_UMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7313 if (RetVT.SimpleTy != MVT::v2i32)
7314 return Register();
7315 if ((Subtarget->hasNEON())) {
7316 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
7317 }
7318 return Register();
7319}
7320
7321Register fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7322 if (RetVT.SimpleTy != MVT::v4i32)
7323 return Register();
7324 if ((Subtarget->hasMVEIntegerOps())) {
7325 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXu32, RC: &ARM::MQPRRegClass, Op0, Op1);
7326 }
7327 if ((Subtarget->hasNEON())) {
7328 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
7329 }
7330 return Register();
7331}
7332
7333Register fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7334 switch (VT.SimpleTy) {
7335 case MVT::v8i8: return fastEmit_ISD_UMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
7336 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
7337 case MVT::v4i16: return fastEmit_ISD_UMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
7338 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
7339 case MVT::v2i32: return fastEmit_ISD_UMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
7340 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
7341 default: return Register();
7342 }
7343}
7344
7345// FastEmit functions for ISD::UMIN.
7346
7347Register fastEmit_ISD_UMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7348 if (RetVT.SimpleTy != MVT::v8i8)
7349 return Register();
7350 if ((Subtarget->hasNEON())) {
7351 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
7352 }
7353 return Register();
7354}
7355
7356Register fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7357 if (RetVT.SimpleTy != MVT::v16i8)
7358 return Register();
7359 if ((Subtarget->hasMVEIntegerOps())) {
7360 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINu8, RC: &ARM::MQPRRegClass, Op0, Op1);
7361 }
7362 if ((Subtarget->hasNEON())) {
7363 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
7364 }
7365 return Register();
7366}
7367
7368Register fastEmit_ISD_UMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7369 if (RetVT.SimpleTy != MVT::v4i16)
7370 return Register();
7371 if ((Subtarget->hasNEON())) {
7372 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
7373 }
7374 return Register();
7375}
7376
7377Register fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7378 if (RetVT.SimpleTy != MVT::v8i16)
7379 return Register();
7380 if ((Subtarget->hasMVEIntegerOps())) {
7381 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINu16, RC: &ARM::MQPRRegClass, Op0, Op1);
7382 }
7383 if ((Subtarget->hasNEON())) {
7384 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
7385 }
7386 return Register();
7387}
7388
7389Register fastEmit_ISD_UMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7390 if (RetVT.SimpleTy != MVT::v2i32)
7391 return Register();
7392 if ((Subtarget->hasNEON())) {
7393 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
7394 }
7395 return Register();
7396}
7397
7398Register fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7399 if (RetVT.SimpleTy != MVT::v4i32)
7400 return Register();
7401 if ((Subtarget->hasMVEIntegerOps())) {
7402 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINu32, RC: &ARM::MQPRRegClass, Op0, Op1);
7403 }
7404 if ((Subtarget->hasNEON())) {
7405 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
7406 }
7407 return Register();
7408}
7409
7410Register fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7411 switch (VT.SimpleTy) {
7412 case MVT::v8i8: return fastEmit_ISD_UMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
7413 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
7414 case MVT::v4i16: return fastEmit_ISD_UMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
7415 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
7416 case MVT::v2i32: return fastEmit_ISD_UMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
7417 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
7418 default: return Register();
7419 }
7420}
7421
7422// FastEmit functions for ISD::USUBSAT.
7423
7424Register fastEmit_ISD_USUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7425 if (RetVT.SimpleTy != MVT::v8i8)
7426 return Register();
7427 if ((Subtarget->hasNEON())) {
7428 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
7429 }
7430 return Register();
7431}
7432
7433Register fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7434 if (RetVT.SimpleTy != MVT::v16i8)
7435 return Register();
7436 if ((Subtarget->hasMVEIntegerOps())) {
7437 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBu8, RC: &ARM::MQPRRegClass, Op0, Op1);
7438 }
7439 if ((Subtarget->hasNEON())) {
7440 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
7441 }
7442 return Register();
7443}
7444
7445Register fastEmit_ISD_USUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7446 if (RetVT.SimpleTy != MVT::v4i16)
7447 return Register();
7448 if ((Subtarget->hasNEON())) {
7449 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
7450 }
7451 return Register();
7452}
7453
7454Register fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7455 if (RetVT.SimpleTy != MVT::v8i16)
7456 return Register();
7457 if ((Subtarget->hasMVEIntegerOps())) {
7458 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBu16, RC: &ARM::MQPRRegClass, Op0, Op1);
7459 }
7460 if ((Subtarget->hasNEON())) {
7461 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
7462 }
7463 return Register();
7464}
7465
7466Register fastEmit_ISD_USUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7467 if (RetVT.SimpleTy != MVT::v2i32)
7468 return Register();
7469 if ((Subtarget->hasNEON())) {
7470 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
7471 }
7472 return Register();
7473}
7474
7475Register fastEmit_ISD_USUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7476 if (RetVT.SimpleTy != MVT::v4i32)
7477 return Register();
7478 if ((Subtarget->hasMVEIntegerOps())) {
7479 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBu32, RC: &ARM::MQPRRegClass, Op0, Op1);
7480 }
7481 if ((Subtarget->hasNEON())) {
7482 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
7483 }
7484 return Register();
7485}
7486
7487Register fastEmit_ISD_USUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7488 if (RetVT.SimpleTy != MVT::v1i64)
7489 return Register();
7490 if ((Subtarget->hasNEON())) {
7491 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
7492 }
7493 return Register();
7494}
7495
7496Register fastEmit_ISD_USUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7497 if (RetVT.SimpleTy != MVT::v2i64)
7498 return Register();
7499 if ((Subtarget->hasNEON())) {
7500 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
7501 }
7502 return Register();
7503}
7504
7505Register fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7506 switch (VT.SimpleTy) {
7507 case MVT::v8i8: return fastEmit_ISD_USUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
7508 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
7509 case MVT::v4i16: return fastEmit_ISD_USUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
7510 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
7511 case MVT::v2i32: return fastEmit_ISD_USUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
7512 case MVT::v4i32: return fastEmit_ISD_USUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
7513 case MVT::v1i64: return fastEmit_ISD_USUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
7514 case MVT::v2i64: return fastEmit_ISD_USUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
7515 default: return Register();
7516 }
7517}
7518
7519// FastEmit functions for ISD::XOR.
7520
7521Register fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
7522 if (RetVT.SimpleTy != MVT::i32)
7523 return Register();
7524 if ((Subtarget->isThumb2())) {
7525 return fastEmitInst_rr(MachineInstOpcode: ARM::t2EORrr, RC: &ARM::rGPRRegClass, Op0, Op1);
7526 }
7527 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
7528 return fastEmitInst_rr(MachineInstOpcode: ARM::tEOR, RC: &ARM::tGPRRegClass, Op0, Op1);
7529 }
7530 if ((!Subtarget->isThumb())) {
7531 return fastEmitInst_rr(MachineInstOpcode: ARM::EORrr, RC: &ARM::GPRRegClass, Op0, Op1);
7532 }
7533 return Register();
7534}
7535
7536Register fastEmit_ISD_XOR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7537 if (RetVT.SimpleTy != MVT::v8i8)
7538 return Register();
7539 if ((Subtarget->hasNEON())) {
7540 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
7541 }
7542 return Register();
7543}
7544
7545Register fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7546 if (RetVT.SimpleTy != MVT::v16i8)
7547 return Register();
7548 if ((Subtarget->hasMVEIntegerOps())) {
7549 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
7550 }
7551 if ((Subtarget->hasNEON())) {
7552 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
7553 }
7554 return Register();
7555}
7556
7557Register fastEmit_ISD_XOR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7558 if (RetVT.SimpleTy != MVT::v4i16)
7559 return Register();
7560 if ((Subtarget->hasNEON())) {
7561 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
7562 }
7563 return Register();
7564}
7565
7566Register fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7567 if (RetVT.SimpleTy != MVT::v8i16)
7568 return Register();
7569 if ((Subtarget->hasMVEIntegerOps())) {
7570 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
7571 }
7572 if ((Subtarget->hasNEON())) {
7573 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
7574 }
7575 return Register();
7576}
7577
7578Register fastEmit_ISD_XOR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7579 if (RetVT.SimpleTy != MVT::v2i32)
7580 return Register();
7581 if ((Subtarget->hasNEON())) {
7582 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
7583 }
7584 return Register();
7585}
7586
7587Register fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7588 if (RetVT.SimpleTy != MVT::v4i32)
7589 return Register();
7590 if ((Subtarget->hasMVEIntegerOps())) {
7591 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
7592 }
7593 if ((Subtarget->hasNEON())) {
7594 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
7595 }
7596 return Register();
7597}
7598
7599Register fastEmit_ISD_XOR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7600 if (RetVT.SimpleTy != MVT::v1i64)
7601 return Register();
7602 if ((Subtarget->hasNEON())) {
7603 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
7604 }
7605 return Register();
7606}
7607
7608Register fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7609 if (RetVT.SimpleTy != MVT::v2i64)
7610 return Register();
7611 if ((Subtarget->hasMVEIntegerOps())) {
7612 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
7613 }
7614 if ((Subtarget->hasNEON())) {
7615 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
7616 }
7617 return Register();
7618}
7619
7620Register fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7621 switch (VT.SimpleTy) {
7622 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
7623 case MVT::v8i8: return fastEmit_ISD_XOR_MVT_v8i8_rr(RetVT, Op0, Op1);
7624 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
7625 case MVT::v4i16: return fastEmit_ISD_XOR_MVT_v4i16_rr(RetVT, Op0, Op1);
7626 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
7627 case MVT::v2i32: return fastEmit_ISD_XOR_MVT_v2i32_rr(RetVT, Op0, Op1);
7628 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
7629 case MVT::v1i64: return fastEmit_ISD_XOR_MVT_v1i64_rr(RetVT, Op0, Op1);
7630 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
7631 default: return Register();
7632 }
7633}
7634
7635// Top-level FastEmit function.
7636
7637Register fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, Register Op1) override {
7638 switch (Opcode) {
7639 case ARMISD::CMN: return fastEmit_ARMISD_CMN_rr(VT, RetVT, Op0, Op1);
7640 case ARMISD::CMP: return fastEmit_ARMISD_CMP_rr(VT, RetVT, Op0, Op1);
7641 case ARMISD::CMPFP: return fastEmit_ARMISD_CMPFP_rr(VT, RetVT, Op0, Op1);
7642 case ARMISD::CMPFPE: return fastEmit_ARMISD_CMPFPE_rr(VT, RetVT, Op0, Op1);
7643 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_rr(VT, RetVT, Op0, Op1);
7644 case ARMISD::EH_SJLJ_LONGJMP: return fastEmit_ARMISD_EH_SJLJ_LONGJMP_rr(VT, RetVT, Op0, Op1);
7645 case ARMISD::EH_SJLJ_SETJMP: return fastEmit_ARMISD_EH_SJLJ_SETJMP_rr(VT, RetVT, Op0, Op1);
7646 case ARMISD::QADD16b: return fastEmit_ARMISD_QADD16b_rr(VT, RetVT, Op0, Op1);
7647 case ARMISD::QADD8b: return fastEmit_ARMISD_QADD8b_rr(VT, RetVT, Op0, Op1);
7648 case ARMISD::QSUB16b: return fastEmit_ARMISD_QSUB16b_rr(VT, RetVT, Op0, Op1);
7649 case ARMISD::QSUB8b: return fastEmit_ARMISD_QSUB8b_rr(VT, RetVT, Op0, Op1);
7650 case ARMISD::SMULWB: return fastEmit_ARMISD_SMULWB_rr(VT, RetVT, Op0, Op1);
7651 case ARMISD::SMULWT: return fastEmit_ARMISD_SMULWT_rr(VT, RetVT, Op0, Op1);
7652 case ARMISD::UQADD16b: return fastEmit_ARMISD_UQADD16b_rr(VT, RetVT, Op0, Op1);
7653 case ARMISD::UQADD8b: return fastEmit_ARMISD_UQADD8b_rr(VT, RetVT, Op0, Op1);
7654 case ARMISD::UQSUB16b: return fastEmit_ARMISD_UQSUB16b_rr(VT, RetVT, Op0, Op1);
7655 case ARMISD::UQSUB8b: return fastEmit_ARMISD_UQSUB8b_rr(VT, RetVT, Op0, Op1);
7656 case ARMISD::VMLAVs: return fastEmit_ARMISD_VMLAVs_rr(VT, RetVT, Op0, Op1);
7657 case ARMISD::VMLAVu: return fastEmit_ARMISD_VMLAVu_rr(VT, RetVT, Op0, Op1);
7658 case ARMISD::VMOVDRR: return fastEmit_ARMISD_VMOVDRR_rr(VT, RetVT, Op0, Op1);
7659 case ARMISD::VMULLs: return fastEmit_ARMISD_VMULLs_rr(VT, RetVT, Op0, Op1);
7660 case ARMISD::VMULLu: return fastEmit_ARMISD_VMULLu_rr(VT, RetVT, Op0, Op1);
7661 case ARMISD::VQDMULH: return fastEmit_ARMISD_VQDMULH_rr(VT, RetVT, Op0, Op1);
7662 case ARMISD::VSHLs: return fastEmit_ARMISD_VSHLs_rr(VT, RetVT, Op0, Op1);
7663 case ARMISD::VSHLu: return fastEmit_ARMISD_VSHLu_rr(VT, RetVT, Op0, Op1);
7664 case ARMISD::VTBL1: return fastEmit_ARMISD_VTBL1_rr(VT, RetVT, Op0, Op1);
7665 case ARMISD::VTST: return fastEmit_ARMISD_VTST_rr(VT, RetVT, Op0, Op1);
7666 case ISD::ABDS: return fastEmit_ISD_ABDS_rr(VT, RetVT, Op0, Op1);
7667 case ISD::ABDU: return fastEmit_ISD_ABDU_rr(VT, RetVT, Op0, Op1);
7668 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
7669 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
7670 case ISD::AVGCEILS: return fastEmit_ISD_AVGCEILS_rr(VT, RetVT, Op0, Op1);
7671 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
7672 case ISD::AVGFLOORS: return fastEmit_ISD_AVGFLOORS_rr(VT, RetVT, Op0, Op1);
7673 case ISD::AVGFLOORU: return fastEmit_ISD_AVGFLOORU_rr(VT, RetVT, Op0, Op1);
7674 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
7675 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
7676 case ISD::FMAXIMUM: return fastEmit_ISD_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
7677 case ISD::FMAXNUM: return fastEmit_ISD_FMAXNUM_rr(VT, RetVT, Op0, Op1);
7678 case ISD::FMINIMUM: return fastEmit_ISD_FMINIMUM_rr(VT, RetVT, Op0, Op1);
7679 case ISD::FMINNUM: return fastEmit_ISD_FMINNUM_rr(VT, RetVT, Op0, Op1);
7680 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
7681 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
7682 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
7683 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
7684 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
7685 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
7686 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
7687 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
7688 case ISD::SDIV: return fastEmit_ISD_SDIV_rr(VT, RetVT, Op0, Op1);
7689 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
7690 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
7691 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
7692 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
7693 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
7694 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
7695 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1);
7696 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1);
7697 case ISD::STRICT_FMAXNUM: return fastEmit_ISD_STRICT_FMAXNUM_rr(VT, RetVT, Op0, Op1);
7698 case ISD::STRICT_FMINNUM: return fastEmit_ISD_STRICT_FMINNUM_rr(VT, RetVT, Op0, Op1);
7699 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1);
7700 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1);
7701 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
7702 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
7703 case ISD::UDIV: return fastEmit_ISD_UDIV_rr(VT, RetVT, Op0, Op1);
7704 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
7705 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
7706 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
7707 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
7708 default: return Register();
7709 }
7710}
7711
7712// FastEmit functions for ARMISD::PIC_ADD.
7713
7714Register fastEmit_ARMISD_PIC_ADD_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7715 if (RetVT.SimpleTy != MVT::i32)
7716 return Register();
7717 if ((Subtarget->isThumb())) {
7718 return fastEmitInst_ri(MachineInstOpcode: ARM::tPICADD, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7719 }
7720 if ((!Subtarget->isThumb())) {
7721 return fastEmitInst_ri(MachineInstOpcode: ARM::PICADD, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7722 }
7723 return Register();
7724}
7725
7726Register fastEmit_ARMISD_PIC_ADD_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7727 switch (VT.SimpleTy) {
7728 case MVT::i32: return fastEmit_ARMISD_PIC_ADD_MVT_i32_ri(RetVT, Op0, imm1);
7729 default: return Register();
7730 }
7731}
7732
7733// FastEmit functions for ARMISD::VDUPLANE.
7734
7735Register fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7736 if (RetVT.SimpleTy != MVT::v8i8)
7737 return Register();
7738 if ((Subtarget->hasNEON())) {
7739 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN8d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7740 }
7741 return Register();
7742}
7743
7744Register fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7745 if (RetVT.SimpleTy != MVT::v4i16)
7746 return Register();
7747 if ((Subtarget->hasNEON())) {
7748 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7749 }
7750 return Register();
7751}
7752
7753Register fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7754 if (RetVT.SimpleTy != MVT::v2i32)
7755 return Register();
7756 if ((Subtarget->hasNEON())) {
7757 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7758 }
7759 return Register();
7760}
7761
7762Register fastEmit_ARMISD_VDUPLANE_MVT_v4f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7763 if (RetVT.SimpleTy != MVT::v4f16)
7764 return Register();
7765 if ((Subtarget->hasNEON())) {
7766 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7767 }
7768 return Register();
7769}
7770
7771Register fastEmit_ARMISD_VDUPLANE_MVT_v4bf16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7772 if (RetVT.SimpleTy != MVT::v4bf16)
7773 return Register();
7774 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
7775 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7776 }
7777 return Register();
7778}
7779
7780Register fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v2f32_ri(Register Op0, uint64_t imm1) {
7781 if ((Subtarget->hasNEON())) {
7782 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7783 }
7784 return Register();
7785}
7786
7787Register fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v4f32_ri(Register Op0, uint64_t imm1) {
7788 if ((Subtarget->hasNEON())) {
7789 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7790 }
7791 return Register();
7792}
7793
7794Register fastEmit_ARMISD_VDUPLANE_MVT_v2f32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7795switch (RetVT.SimpleTy) {
7796 case MVT::v2f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v2f32_ri(Op0, imm1);
7797 case MVT::v4f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v4f32_ri(Op0, imm1);
7798 default: return Register();
7799}
7800}
7801
7802Register fastEmit_ARMISD_VDUPLANE_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7803 switch (VT.SimpleTy) {
7804 case MVT::v8i8: return fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri(RetVT, Op0, imm1);
7805 case MVT::v4i16: return fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri(RetVT, Op0, imm1);
7806 case MVT::v2i32: return fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri(RetVT, Op0, imm1);
7807 case MVT::v4f16: return fastEmit_ARMISD_VDUPLANE_MVT_v4f16_ri(RetVT, Op0, imm1);
7808 case MVT::v4bf16: return fastEmit_ARMISD_VDUPLANE_MVT_v4bf16_ri(RetVT, Op0, imm1);
7809 case MVT::v2f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_ri(RetVT, Op0, imm1);
7810 default: return Register();
7811 }
7812}
7813
7814// FastEmit functions for ARMISD::VGETLANEs.
7815
7816Register fastEmit_ARMISD_VGETLANEs_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7817 if (RetVT.SimpleTy != MVT::i32)
7818 return Register();
7819 if ((Subtarget->hasNEON())) {
7820 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNs8, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7821 }
7822 return Register();
7823}
7824
7825Register fastEmit_ARMISD_VGETLANEs_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7826 if (RetVT.SimpleTy != MVT::i32)
7827 return Register();
7828 if ((Subtarget->hasMVEIntegerOps())) {
7829 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_s8, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7830 }
7831 return Register();
7832}
7833
7834Register fastEmit_ARMISD_VGETLANEs_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7835 if (RetVT.SimpleTy != MVT::i32)
7836 return Register();
7837 if ((Subtarget->hasNEON())) {
7838 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNs16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7839 }
7840 return Register();
7841}
7842
7843Register fastEmit_ARMISD_VGETLANEs_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7844 if (RetVT.SimpleTy != MVT::i32)
7845 return Register();
7846 if ((Subtarget->hasMVEIntegerOps())) {
7847 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_s16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7848 }
7849 return Register();
7850}
7851
7852Register fastEmit_ARMISD_VGETLANEs_MVT_v8f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7853 if (RetVT.SimpleTy != MVT::i32)
7854 return Register();
7855 if ((Subtarget->hasMVEIntegerOps())) {
7856 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_s16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7857 }
7858 return Register();
7859}
7860
7861Register fastEmit_ARMISD_VGETLANEs_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7862 switch (VT.SimpleTy) {
7863 case MVT::v8i8: return fastEmit_ARMISD_VGETLANEs_MVT_v8i8_ri(RetVT, Op0, imm1);
7864 case MVT::v16i8: return fastEmit_ARMISD_VGETLANEs_MVT_v16i8_ri(RetVT, Op0, imm1);
7865 case MVT::v4i16: return fastEmit_ARMISD_VGETLANEs_MVT_v4i16_ri(RetVT, Op0, imm1);
7866 case MVT::v8i16: return fastEmit_ARMISD_VGETLANEs_MVT_v8i16_ri(RetVT, Op0, imm1);
7867 case MVT::v8f16: return fastEmit_ARMISD_VGETLANEs_MVT_v8f16_ri(RetVT, Op0, imm1);
7868 default: return Register();
7869 }
7870}
7871
7872// FastEmit functions for ARMISD::VGETLANEu.
7873
7874Register fastEmit_ARMISD_VGETLANEu_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7875 if (RetVT.SimpleTy != MVT::i32)
7876 return Register();
7877 if ((Subtarget->hasNEON())) {
7878 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu8, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7879 }
7880 return Register();
7881}
7882
7883Register fastEmit_ARMISD_VGETLANEu_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7884 if (RetVT.SimpleTy != MVT::i32)
7885 return Register();
7886 if ((Subtarget->hasMVEIntegerOps())) {
7887 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_u8, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7888 }
7889 return Register();
7890}
7891
7892Register fastEmit_ARMISD_VGETLANEu_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7893 if (RetVT.SimpleTy != MVT::i32)
7894 return Register();
7895 if ((Subtarget->hasNEON())) {
7896 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7897 }
7898 return Register();
7899}
7900
7901Register fastEmit_ARMISD_VGETLANEu_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7902 if (RetVT.SimpleTy != MVT::i32)
7903 return Register();
7904 if ((Subtarget->hasMVEIntegerOps())) {
7905 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_u16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7906 }
7907 return Register();
7908}
7909
7910Register fastEmit_ARMISD_VGETLANEu_MVT_v4f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7911 if (RetVT.SimpleTy != MVT::i32)
7912 return Register();
7913 if ((Subtarget->hasNEON())) {
7914 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7915 }
7916 return Register();
7917}
7918
7919Register fastEmit_ARMISD_VGETLANEu_MVT_v8f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7920 if (RetVT.SimpleTy != MVT::i32)
7921 return Register();
7922 if ((Subtarget->hasMVEIntegerOps())) {
7923 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_u16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7924 }
7925 return Register();
7926}
7927
7928Register fastEmit_ARMISD_VGETLANEu_MVT_v4bf16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7929 if (RetVT.SimpleTy != MVT::i32)
7930 return Register();
7931 if ((Subtarget->hasNEON())) {
7932 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7933 }
7934 return Register();
7935}
7936
7937Register fastEmit_ARMISD_VGETLANEu_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7938 switch (VT.SimpleTy) {
7939 case MVT::v8i8: return fastEmit_ARMISD_VGETLANEu_MVT_v8i8_ri(RetVT, Op0, imm1);
7940 case MVT::v16i8: return fastEmit_ARMISD_VGETLANEu_MVT_v16i8_ri(RetVT, Op0, imm1);
7941 case MVT::v4i16: return fastEmit_ARMISD_VGETLANEu_MVT_v4i16_ri(RetVT, Op0, imm1);
7942 case MVT::v8i16: return fastEmit_ARMISD_VGETLANEu_MVT_v8i16_ri(RetVT, Op0, imm1);
7943 case MVT::v4f16: return fastEmit_ARMISD_VGETLANEu_MVT_v4f16_ri(RetVT, Op0, imm1);
7944 case MVT::v8f16: return fastEmit_ARMISD_VGETLANEu_MVT_v8f16_ri(RetVT, Op0, imm1);
7945 case MVT::v4bf16: return fastEmit_ARMISD_VGETLANEu_MVT_v4bf16_ri(RetVT, Op0, imm1);
7946 default: return Register();
7947 }
7948}
7949
7950// FastEmit functions for ARMISD::VQSHLsIMM.
7951
7952Register fastEmit_ARMISD_VQSHLsIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7953 if (RetVT.SimpleTy != MVT::v8i8)
7954 return Register();
7955 if ((Subtarget->hasNEON())) {
7956 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7957 }
7958 return Register();
7959}
7960
7961Register fastEmit_ARMISD_VQSHLsIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7962 if (RetVT.SimpleTy != MVT::v16i8)
7963 return Register();
7964 if ((Subtarget->hasNEON())) {
7965 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7966 }
7967 return Register();
7968}
7969
7970Register fastEmit_ARMISD_VQSHLsIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7971 if (RetVT.SimpleTy != MVT::v4i16)
7972 return Register();
7973 if ((Subtarget->hasNEON())) {
7974 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7975 }
7976 return Register();
7977}
7978
7979Register fastEmit_ARMISD_VQSHLsIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7980 if (RetVT.SimpleTy != MVT::v8i16)
7981 return Register();
7982 if ((Subtarget->hasNEON())) {
7983 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7984 }
7985 return Register();
7986}
7987
7988Register fastEmit_ARMISD_VQSHLsIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7989 if (RetVT.SimpleTy != MVT::v2i32)
7990 return Register();
7991 if ((Subtarget->hasNEON())) {
7992 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7993 }
7994 return Register();
7995}
7996
7997Register fastEmit_ARMISD_VQSHLsIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7998 if (RetVT.SimpleTy != MVT::v4i32)
7999 return Register();
8000 if ((Subtarget->hasNEON())) {
8001 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8002 }
8003 return Register();
8004}
8005
8006Register fastEmit_ARMISD_VQSHLsIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8007 if (RetVT.SimpleTy != MVT::v1i64)
8008 return Register();
8009 if ((Subtarget->hasNEON())) {
8010 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8011 }
8012 return Register();
8013}
8014
8015Register fastEmit_ARMISD_VQSHLsIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8016 if (RetVT.SimpleTy != MVT::v2i64)
8017 return Register();
8018 if ((Subtarget->hasNEON())) {
8019 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8020 }
8021 return Register();
8022}
8023
8024Register fastEmit_ARMISD_VQSHLsIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8025 switch (VT.SimpleTy) {
8026 case MVT::v8i8: return fastEmit_ARMISD_VQSHLsIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8027 case MVT::v16i8: return fastEmit_ARMISD_VQSHLsIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8028 case MVT::v4i16: return fastEmit_ARMISD_VQSHLsIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8029 case MVT::v8i16: return fastEmit_ARMISD_VQSHLsIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8030 case MVT::v2i32: return fastEmit_ARMISD_VQSHLsIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8031 case MVT::v4i32: return fastEmit_ARMISD_VQSHLsIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8032 case MVT::v1i64: return fastEmit_ARMISD_VQSHLsIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8033 case MVT::v2i64: return fastEmit_ARMISD_VQSHLsIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8034 default: return Register();
8035 }
8036}
8037
8038// FastEmit functions for ARMISD::VQSHLsuIMM.
8039
8040Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8041 if (RetVT.SimpleTy != MVT::v8i8)
8042 return Register();
8043 if ((Subtarget->hasNEON())) {
8044 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8045 }
8046 return Register();
8047}
8048
8049Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8050 if (RetVT.SimpleTy != MVT::v16i8)
8051 return Register();
8052 if ((Subtarget->hasNEON())) {
8053 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8054 }
8055 return Register();
8056}
8057
8058Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8059 if (RetVT.SimpleTy != MVT::v4i16)
8060 return Register();
8061 if ((Subtarget->hasNEON())) {
8062 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8063 }
8064 return Register();
8065}
8066
8067Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8068 if (RetVT.SimpleTy != MVT::v8i16)
8069 return Register();
8070 if ((Subtarget->hasNEON())) {
8071 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8072 }
8073 return Register();
8074}
8075
8076Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8077 if (RetVT.SimpleTy != MVT::v2i32)
8078 return Register();
8079 if ((Subtarget->hasNEON())) {
8080 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8081 }
8082 return Register();
8083}
8084
8085Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8086 if (RetVT.SimpleTy != MVT::v4i32)
8087 return Register();
8088 if ((Subtarget->hasNEON())) {
8089 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8090 }
8091 return Register();
8092}
8093
8094Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8095 if (RetVT.SimpleTy != MVT::v1i64)
8096 return Register();
8097 if ((Subtarget->hasNEON())) {
8098 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8099 }
8100 return Register();
8101}
8102
8103Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8104 if (RetVT.SimpleTy != MVT::v2i64)
8105 return Register();
8106 if ((Subtarget->hasNEON())) {
8107 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8108 }
8109 return Register();
8110}
8111
8112Register fastEmit_ARMISD_VQSHLsuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8113 switch (VT.SimpleTy) {
8114 case MVT::v8i8: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8115 case MVT::v16i8: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8116 case MVT::v4i16: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8117 case MVT::v8i16: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8118 case MVT::v2i32: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8119 case MVT::v4i32: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8120 case MVT::v1i64: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8121 case MVT::v2i64: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8122 default: return Register();
8123 }
8124}
8125
8126// FastEmit functions for ARMISD::VQSHLuIMM.
8127
8128Register fastEmit_ARMISD_VQSHLuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8129 if (RetVT.SimpleTy != MVT::v8i8)
8130 return Register();
8131 if ((Subtarget->hasNEON())) {
8132 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8133 }
8134 return Register();
8135}
8136
8137Register fastEmit_ARMISD_VQSHLuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8138 if (RetVT.SimpleTy != MVT::v16i8)
8139 return Register();
8140 if ((Subtarget->hasNEON())) {
8141 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8142 }
8143 return Register();
8144}
8145
8146Register fastEmit_ARMISD_VQSHLuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8147 if (RetVT.SimpleTy != MVT::v4i16)
8148 return Register();
8149 if ((Subtarget->hasNEON())) {
8150 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8151 }
8152 return Register();
8153}
8154
8155Register fastEmit_ARMISD_VQSHLuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8156 if (RetVT.SimpleTy != MVT::v8i16)
8157 return Register();
8158 if ((Subtarget->hasNEON())) {
8159 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8160 }
8161 return Register();
8162}
8163
8164Register fastEmit_ARMISD_VQSHLuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8165 if (RetVT.SimpleTy != MVT::v2i32)
8166 return Register();
8167 if ((Subtarget->hasNEON())) {
8168 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8169 }
8170 return Register();
8171}
8172
8173Register fastEmit_ARMISD_VQSHLuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8174 if (RetVT.SimpleTy != MVT::v4i32)
8175 return Register();
8176 if ((Subtarget->hasNEON())) {
8177 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8178 }
8179 return Register();
8180}
8181
8182Register fastEmit_ARMISD_VQSHLuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8183 if (RetVT.SimpleTy != MVT::v1i64)
8184 return Register();
8185 if ((Subtarget->hasNEON())) {
8186 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8187 }
8188 return Register();
8189}
8190
8191Register fastEmit_ARMISD_VQSHLuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8192 if (RetVT.SimpleTy != MVT::v2i64)
8193 return Register();
8194 if ((Subtarget->hasNEON())) {
8195 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8196 }
8197 return Register();
8198}
8199
8200Register fastEmit_ARMISD_VQSHLuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8201 switch (VT.SimpleTy) {
8202 case MVT::v8i8: return fastEmit_ARMISD_VQSHLuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8203 case MVT::v16i8: return fastEmit_ARMISD_VQSHLuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8204 case MVT::v4i16: return fastEmit_ARMISD_VQSHLuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8205 case MVT::v8i16: return fastEmit_ARMISD_VQSHLuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8206 case MVT::v2i32: return fastEmit_ARMISD_VQSHLuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8207 case MVT::v4i32: return fastEmit_ARMISD_VQSHLuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8208 case MVT::v1i64: return fastEmit_ARMISD_VQSHLuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8209 case MVT::v2i64: return fastEmit_ARMISD_VQSHLuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8210 default: return Register();
8211 }
8212}
8213
8214// FastEmit functions for ARMISD::VRSHRsIMM.
8215
8216Register fastEmit_ARMISD_VRSHRsIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8217 if (RetVT.SimpleTy != MVT::v8i8)
8218 return Register();
8219 if ((Subtarget->hasNEON())) {
8220 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8221 }
8222 return Register();
8223}
8224
8225Register fastEmit_ARMISD_VRSHRsIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8226 if (RetVT.SimpleTy != MVT::v16i8)
8227 return Register();
8228 if ((Subtarget->hasNEON())) {
8229 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8230 }
8231 return Register();
8232}
8233
8234Register fastEmit_ARMISD_VRSHRsIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8235 if (RetVT.SimpleTy != MVT::v4i16)
8236 return Register();
8237 if ((Subtarget->hasNEON())) {
8238 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8239 }
8240 return Register();
8241}
8242
8243Register fastEmit_ARMISD_VRSHRsIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8244 if (RetVT.SimpleTy != MVT::v8i16)
8245 return Register();
8246 if ((Subtarget->hasNEON())) {
8247 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8248 }
8249 return Register();
8250}
8251
8252Register fastEmit_ARMISD_VRSHRsIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8253 if (RetVT.SimpleTy != MVT::v2i32)
8254 return Register();
8255 if ((Subtarget->hasNEON())) {
8256 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8257 }
8258 return Register();
8259}
8260
8261Register fastEmit_ARMISD_VRSHRsIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8262 if (RetVT.SimpleTy != MVT::v4i32)
8263 return Register();
8264 if ((Subtarget->hasNEON())) {
8265 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8266 }
8267 return Register();
8268}
8269
8270Register fastEmit_ARMISD_VRSHRsIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8271 if (RetVT.SimpleTy != MVT::v1i64)
8272 return Register();
8273 if ((Subtarget->hasNEON())) {
8274 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8275 }
8276 return Register();
8277}
8278
8279Register fastEmit_ARMISD_VRSHRsIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8280 if (RetVT.SimpleTy != MVT::v2i64)
8281 return Register();
8282 if ((Subtarget->hasNEON())) {
8283 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8284 }
8285 return Register();
8286}
8287
8288Register fastEmit_ARMISD_VRSHRsIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8289 switch (VT.SimpleTy) {
8290 case MVT::v8i8: return fastEmit_ARMISD_VRSHRsIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8291 case MVT::v16i8: return fastEmit_ARMISD_VRSHRsIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8292 case MVT::v4i16: return fastEmit_ARMISD_VRSHRsIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8293 case MVT::v8i16: return fastEmit_ARMISD_VRSHRsIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8294 case MVT::v2i32: return fastEmit_ARMISD_VRSHRsIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8295 case MVT::v4i32: return fastEmit_ARMISD_VRSHRsIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8296 case MVT::v1i64: return fastEmit_ARMISD_VRSHRsIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8297 case MVT::v2i64: return fastEmit_ARMISD_VRSHRsIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8298 default: return Register();
8299 }
8300}
8301
8302// FastEmit functions for ARMISD::VRSHRuIMM.
8303
8304Register fastEmit_ARMISD_VRSHRuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8305 if (RetVT.SimpleTy != MVT::v8i8)
8306 return Register();
8307 if ((Subtarget->hasNEON())) {
8308 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8309 }
8310 return Register();
8311}
8312
8313Register fastEmit_ARMISD_VRSHRuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8314 if (RetVT.SimpleTy != MVT::v16i8)
8315 return Register();
8316 if ((Subtarget->hasNEON())) {
8317 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8318 }
8319 return Register();
8320}
8321
8322Register fastEmit_ARMISD_VRSHRuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8323 if (RetVT.SimpleTy != MVT::v4i16)
8324 return Register();
8325 if ((Subtarget->hasNEON())) {
8326 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8327 }
8328 return Register();
8329}
8330
8331Register fastEmit_ARMISD_VRSHRuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8332 if (RetVT.SimpleTy != MVT::v8i16)
8333 return Register();
8334 if ((Subtarget->hasNEON())) {
8335 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8336 }
8337 return Register();
8338}
8339
8340Register fastEmit_ARMISD_VRSHRuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8341 if (RetVT.SimpleTy != MVT::v2i32)
8342 return Register();
8343 if ((Subtarget->hasNEON())) {
8344 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8345 }
8346 return Register();
8347}
8348
8349Register fastEmit_ARMISD_VRSHRuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8350 if (RetVT.SimpleTy != MVT::v4i32)
8351 return Register();
8352 if ((Subtarget->hasNEON())) {
8353 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8354 }
8355 return Register();
8356}
8357
8358Register fastEmit_ARMISD_VRSHRuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8359 if (RetVT.SimpleTy != MVT::v1i64)
8360 return Register();
8361 if ((Subtarget->hasNEON())) {
8362 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8363 }
8364 return Register();
8365}
8366
8367Register fastEmit_ARMISD_VRSHRuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8368 if (RetVT.SimpleTy != MVT::v2i64)
8369 return Register();
8370 if ((Subtarget->hasNEON())) {
8371 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8372 }
8373 return Register();
8374}
8375
8376Register fastEmit_ARMISD_VRSHRuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8377 switch (VT.SimpleTy) {
8378 case MVT::v8i8: return fastEmit_ARMISD_VRSHRuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8379 case MVT::v16i8: return fastEmit_ARMISD_VRSHRuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8380 case MVT::v4i16: return fastEmit_ARMISD_VRSHRuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8381 case MVT::v8i16: return fastEmit_ARMISD_VRSHRuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8382 case MVT::v2i32: return fastEmit_ARMISD_VRSHRuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8383 case MVT::v4i32: return fastEmit_ARMISD_VRSHRuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8384 case MVT::v1i64: return fastEmit_ARMISD_VRSHRuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8385 case MVT::v2i64: return fastEmit_ARMISD_VRSHRuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8386 default: return Register();
8387 }
8388}
8389
8390// FastEmit functions for ARMISD::VSHLIMM.
8391
8392Register fastEmit_ARMISD_VSHLIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8393 if (RetVT.SimpleTy != MVT::v8i8)
8394 return Register();
8395 if ((Subtarget->hasNEON())) {
8396 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8397 }
8398 return Register();
8399}
8400
8401Register fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8402 if (RetVT.SimpleTy != MVT::v16i8)
8403 return Register();
8404 if ((Subtarget->hasNEON())) {
8405 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8406 }
8407 return Register();
8408}
8409
8410Register fastEmit_ARMISD_VSHLIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8411 if (RetVT.SimpleTy != MVT::v4i16)
8412 return Register();
8413 if ((Subtarget->hasNEON())) {
8414 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8415 }
8416 return Register();
8417}
8418
8419Register fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8420 if (RetVT.SimpleTy != MVT::v8i16)
8421 return Register();
8422 if ((Subtarget->hasNEON())) {
8423 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8424 }
8425 return Register();
8426}
8427
8428Register fastEmit_ARMISD_VSHLIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8429 if (RetVT.SimpleTy != MVT::v2i32)
8430 return Register();
8431 if ((Subtarget->hasNEON())) {
8432 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8433 }
8434 return Register();
8435}
8436
8437Register fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8438 if (RetVT.SimpleTy != MVT::v4i32)
8439 return Register();
8440 if ((Subtarget->hasNEON())) {
8441 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8442 }
8443 return Register();
8444}
8445
8446Register fastEmit_ARMISD_VSHLIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8447 if (RetVT.SimpleTy != MVT::v1i64)
8448 return Register();
8449 if ((Subtarget->hasNEON())) {
8450 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8451 }
8452 return Register();
8453}
8454
8455Register fastEmit_ARMISD_VSHLIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8456 if (RetVT.SimpleTy != MVT::v2i64)
8457 return Register();
8458 if ((Subtarget->hasNEON())) {
8459 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8460 }
8461 return Register();
8462}
8463
8464Register fastEmit_ARMISD_VSHLIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8465 switch (VT.SimpleTy) {
8466 case MVT::v8i8: return fastEmit_ARMISD_VSHLIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8467 case MVT::v16i8: return fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8468 case MVT::v4i16: return fastEmit_ARMISD_VSHLIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8469 case MVT::v8i16: return fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8470 case MVT::v2i32: return fastEmit_ARMISD_VSHLIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8471 case MVT::v4i32: return fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8472 case MVT::v1i64: return fastEmit_ARMISD_VSHLIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8473 case MVT::v2i64: return fastEmit_ARMISD_VSHLIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8474 default: return Register();
8475 }
8476}
8477
8478// FastEmit functions for ARMISD::VSHRsIMM.
8479
8480Register fastEmit_ARMISD_VSHRsIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8481 if (RetVT.SimpleTy != MVT::v8i8)
8482 return Register();
8483 if ((Subtarget->hasNEON())) {
8484 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8485 }
8486 return Register();
8487}
8488
8489Register fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8490 if (RetVT.SimpleTy != MVT::v16i8)
8491 return Register();
8492 if ((Subtarget->hasNEON())) {
8493 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8494 }
8495 return Register();
8496}
8497
8498Register fastEmit_ARMISD_VSHRsIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8499 if (RetVT.SimpleTy != MVT::v4i16)
8500 return Register();
8501 if ((Subtarget->hasNEON())) {
8502 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8503 }
8504 return Register();
8505}
8506
8507Register fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8508 if (RetVT.SimpleTy != MVT::v8i16)
8509 return Register();
8510 if ((Subtarget->hasNEON())) {
8511 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8512 }
8513 return Register();
8514}
8515
8516Register fastEmit_ARMISD_VSHRsIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8517 if (RetVT.SimpleTy != MVT::v2i32)
8518 return Register();
8519 if ((Subtarget->hasNEON())) {
8520 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8521 }
8522 return Register();
8523}
8524
8525Register fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8526 if (RetVT.SimpleTy != MVT::v4i32)
8527 return Register();
8528 if ((Subtarget->hasNEON())) {
8529 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8530 }
8531 return Register();
8532}
8533
8534Register fastEmit_ARMISD_VSHRsIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8535 if (RetVT.SimpleTy != MVT::v1i64)
8536 return Register();
8537 if ((Subtarget->hasNEON())) {
8538 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8539 }
8540 return Register();
8541}
8542
8543Register fastEmit_ARMISD_VSHRsIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8544 if (RetVT.SimpleTy != MVT::v2i64)
8545 return Register();
8546 if ((Subtarget->hasNEON())) {
8547 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8548 }
8549 return Register();
8550}
8551
8552Register fastEmit_ARMISD_VSHRsIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8553 switch (VT.SimpleTy) {
8554 case MVT::v8i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8555 case MVT::v16i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8556 case MVT::v4i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8557 case MVT::v8i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8558 case MVT::v2i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8559 case MVT::v4i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8560 case MVT::v1i64: return fastEmit_ARMISD_VSHRsIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8561 case MVT::v2i64: return fastEmit_ARMISD_VSHRsIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8562 default: return Register();
8563 }
8564}
8565
8566// FastEmit functions for ARMISD::VSHRuIMM.
8567
8568Register fastEmit_ARMISD_VSHRuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8569 if (RetVT.SimpleTy != MVT::v8i8)
8570 return Register();
8571 if ((Subtarget->hasNEON())) {
8572 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8573 }
8574 return Register();
8575}
8576
8577Register fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8578 if (RetVT.SimpleTy != MVT::v16i8)
8579 return Register();
8580 if ((Subtarget->hasNEON())) {
8581 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8582 }
8583 return Register();
8584}
8585
8586Register fastEmit_ARMISD_VSHRuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8587 if (RetVT.SimpleTy != MVT::v4i16)
8588 return Register();
8589 if ((Subtarget->hasNEON())) {
8590 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8591 }
8592 return Register();
8593}
8594
8595Register fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8596 if (RetVT.SimpleTy != MVT::v8i16)
8597 return Register();
8598 if ((Subtarget->hasNEON())) {
8599 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8600 }
8601 return Register();
8602}
8603
8604Register fastEmit_ARMISD_VSHRuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8605 if (RetVT.SimpleTy != MVT::v2i32)
8606 return Register();
8607 if ((Subtarget->hasNEON())) {
8608 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8609 }
8610 return Register();
8611}
8612
8613Register fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8614 if (RetVT.SimpleTy != MVT::v4i32)
8615 return Register();
8616 if ((Subtarget->hasNEON())) {
8617 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8618 }
8619 return Register();
8620}
8621
8622Register fastEmit_ARMISD_VSHRuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8623 if (RetVT.SimpleTy != MVT::v1i64)
8624 return Register();
8625 if ((Subtarget->hasNEON())) {
8626 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8627 }
8628 return Register();
8629}
8630
8631Register fastEmit_ARMISD_VSHRuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8632 if (RetVT.SimpleTy != MVT::v2i64)
8633 return Register();
8634 if ((Subtarget->hasNEON())) {
8635 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8636 }
8637 return Register();
8638}
8639
8640Register fastEmit_ARMISD_VSHRuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8641 switch (VT.SimpleTy) {
8642 case MVT::v8i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8643 case MVT::v16i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8644 case MVT::v4i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8645 case MVT::v8i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8646 case MVT::v2i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8647 case MVT::v4i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8648 case MVT::v1i64: return fastEmit_ARMISD_VSHRuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8649 case MVT::v2i64: return fastEmit_ARMISD_VSHRuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8650 default: return Register();
8651 }
8652}
8653
8654// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
8655
8656Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8657 if (RetVT.SimpleTy != MVT::i32)
8658 return Register();
8659 if ((Subtarget->hasFPRegs()) && (!Subtarget->hasSlowVGETLNi32())) {
8660 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNi32, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8661 }
8662 return Register();
8663}
8664
8665Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8666 switch (VT.SimpleTy) {
8667 case MVT::v2i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i32_ri(RetVT, Op0, imm1);
8668 default: return Register();
8669 }
8670}
8671
8672// FastEmit functions for ISD::SHL.
8673
8674Register fastEmit_ISD_SHL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8675 if (RetVT.SimpleTy != MVT::i32)
8676 return Register();
8677 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
8678 return fastEmitInst_ri(MachineInstOpcode: ARM::tLSLri, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
8679 }
8680 return Register();
8681}
8682
8683Register fastEmit_ISD_SHL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8684 switch (VT.SimpleTy) {
8685 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri(RetVT, Op0, imm1);
8686 default: return Register();
8687 }
8688}
8689
8690// Top-level FastEmit function.
8691
8692Register fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) override {
8693 if (VT == MVT::i32 && Predicate_mod_imm(Imm: imm1))
8694 if (Register Reg = fastEmit_ri_Predicate_mod_imm(VT, RetVT, Opcode, Op0, imm1))
8695 return Reg;
8696
8697 if (VT == MVT::i32 && Predicate_imm0_7(Imm: imm1))
8698 if (Register Reg = fastEmit_ri_Predicate_imm0_7(VT, RetVT, Opcode, Op0, imm1))
8699 return Reg;
8700
8701 if (VT == MVT::i32 && Predicate_imm0_255_expr(Imm: imm1))
8702 if (Register Reg = fastEmit_ri_Predicate_imm0_255_expr(VT, RetVT, Opcode, Op0, imm1))
8703 return Reg;
8704
8705 if (VT == MVT::i32 && Predicate_imm0_255(Imm: imm1))
8706 if (Register Reg = fastEmit_ri_Predicate_imm0_255(VT, RetVT, Opcode, Op0, imm1))
8707 return Reg;
8708
8709 if (VT == MVT::i32 && Predicate_t2_so_imm(Imm: imm1))
8710 if (Register Reg = fastEmit_ri_Predicate_t2_so_imm(VT, RetVT, Opcode, Op0, imm1))
8711 return Reg;
8712
8713 if (VT == MVT::i32 && Predicate_imm0_4095(Imm: imm1))
8714 if (Register Reg = fastEmit_ri_Predicate_imm0_4095(VT, RetVT, Opcode, Op0, imm1))
8715 return Reg;
8716
8717 if (VT == MVT::i32 && Predicate_imm1_31(Imm: imm1))
8718 if (Register Reg = fastEmit_ri_Predicate_imm1_31(VT, RetVT, Opcode, Op0, imm1))
8719 return Reg;
8720
8721 if (VT == MVT::i32 && Predicate_shr_imm8(Imm: imm1))
8722 if (Register Reg = fastEmit_ri_Predicate_shr_imm8(VT, RetVT, Opcode, Op0, imm1))
8723 return Reg;
8724
8725 if (VT == MVT::i32 && Predicate_shr_imm16(Imm: imm1))
8726 if (Register Reg = fastEmit_ri_Predicate_shr_imm16(VT, RetVT, Opcode, Op0, imm1))
8727 return Reg;
8728
8729 if (VT == MVT::i32 && Predicate_shr_imm32(Imm: imm1))
8730 if (Register Reg = fastEmit_ri_Predicate_shr_imm32(VT, RetVT, Opcode, Op0, imm1))
8731 return Reg;
8732
8733 if (VT == MVT::i32 && Predicate_VectorIndex32(Imm: imm1))
8734 if (Register Reg = fastEmit_ri_Predicate_VectorIndex32(VT, RetVT, Opcode, Op0, imm1))
8735 return Reg;
8736
8737 if (VT == MVT::i32 && Predicate_imm0_31(Imm: imm1))
8738 if (Register Reg = fastEmit_ri_Predicate_imm0_31(VT, RetVT, Opcode, Op0, imm1))
8739 return Reg;
8740
8741 if (VT == MVT::i32 && Predicate_imm0_15(Imm: imm1))
8742 if (Register Reg = fastEmit_ri_Predicate_imm0_15(VT, RetVT, Opcode, Op0, imm1))
8743 return Reg;
8744
8745 switch (Opcode) {
8746 case ARMISD::PIC_ADD: return fastEmit_ARMISD_PIC_ADD_ri(VT, RetVT, Op0, imm1);
8747 case ARMISD::VDUPLANE: return fastEmit_ARMISD_VDUPLANE_ri(VT, RetVT, Op0, imm1);
8748 case ARMISD::VGETLANEs: return fastEmit_ARMISD_VGETLANEs_ri(VT, RetVT, Op0, imm1);
8749 case ARMISD::VGETLANEu: return fastEmit_ARMISD_VGETLANEu_ri(VT, RetVT, Op0, imm1);
8750 case ARMISD::VQSHLsIMM: return fastEmit_ARMISD_VQSHLsIMM_ri(VT, RetVT, Op0, imm1);
8751 case ARMISD::VQSHLsuIMM: return fastEmit_ARMISD_VQSHLsuIMM_ri(VT, RetVT, Op0, imm1);
8752 case ARMISD::VQSHLuIMM: return fastEmit_ARMISD_VQSHLuIMM_ri(VT, RetVT, Op0, imm1);
8753 case ARMISD::VRSHRsIMM: return fastEmit_ARMISD_VRSHRsIMM_ri(VT, RetVT, Op0, imm1);
8754 case ARMISD::VRSHRuIMM: return fastEmit_ARMISD_VRSHRuIMM_ri(VT, RetVT, Op0, imm1);
8755 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri(VT, RetVT, Op0, imm1);
8756 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri(VT, RetVT, Op0, imm1);
8757 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri(VT, RetVT, Op0, imm1);
8758 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(VT, RetVT, Op0, imm1);
8759 case ISD::SHL: return fastEmit_ISD_SHL_ri(VT, RetVT, Op0, imm1);
8760 default: return Register();
8761 }
8762}
8763
8764// FastEmit functions for ARMISD::CMN.
8765
8766Register fastEmit_ARMISD_CMN_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8767 if (RetVT.SimpleTy != MVT::i32)
8768 return Register();
8769 if ((!Subtarget->isThumb())) {
8770 return fastEmitInst_ri(MachineInstOpcode: ARM::CMNri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8771 }
8772 return Register();
8773}
8774
8775Register fastEmit_ARMISD_CMN_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8776 switch (VT.SimpleTy) {
8777 case MVT::i32: return fastEmit_ARMISD_CMN_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8778 default: return Register();
8779 }
8780}
8781
8782// FastEmit functions for ARMISD::CMP.
8783
8784Register fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8785 if (RetVT.SimpleTy != MVT::i32)
8786 return Register();
8787 if ((!Subtarget->isThumb())) {
8788 return fastEmitInst_ri(MachineInstOpcode: ARM::CMPri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8789 }
8790 return Register();
8791}
8792
8793Register fastEmit_ARMISD_CMP_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8794 switch (VT.SimpleTy) {
8795 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8796 default: return Register();
8797 }
8798}
8799
8800// FastEmit functions for ARMISD::CMPZ.
8801
8802Register fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8803 if (RetVT.SimpleTy != MVT::i32)
8804 return Register();
8805 if ((!Subtarget->isThumb())) {
8806 return fastEmitInst_ri(MachineInstOpcode: ARM::CMPri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8807 }
8808 return Register();
8809}
8810
8811Register fastEmit_ARMISD_CMPZ_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8812 switch (VT.SimpleTy) {
8813 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8814 default: return Register();
8815 }
8816}
8817
8818// FastEmit functions for ISD::ADD.
8819
8820Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8821 if (RetVT.SimpleTy != MVT::i32)
8822 return Register();
8823 if ((!Subtarget->isThumb())) {
8824 return fastEmitInst_ri(MachineInstOpcode: ARM::ADDri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8825 }
8826 return Register();
8827}
8828
8829Register fastEmit_ISD_ADD_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8830 switch (VT.SimpleTy) {
8831 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8832 default: return Register();
8833 }
8834}
8835
8836// FastEmit functions for ISD::AND.
8837
8838Register fastEmit_ISD_AND_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8839 if (RetVT.SimpleTy != MVT::i32)
8840 return Register();
8841 if ((!Subtarget->isThumb())) {
8842 return fastEmitInst_ri(MachineInstOpcode: ARM::ANDri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8843 }
8844 return Register();
8845}
8846
8847Register fastEmit_ISD_AND_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8848 switch (VT.SimpleTy) {
8849 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8850 default: return Register();
8851 }
8852}
8853
8854// FastEmit functions for ISD::OR.
8855
8856Register fastEmit_ISD_OR_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8857 if (RetVT.SimpleTy != MVT::i32)
8858 return Register();
8859 if ((!Subtarget->isThumb())) {
8860 return fastEmitInst_ri(MachineInstOpcode: ARM::ORRri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8861 }
8862 return Register();
8863}
8864
8865Register fastEmit_ISD_OR_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8866 switch (VT.SimpleTy) {
8867 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8868 default: return Register();
8869 }
8870}
8871
8872// FastEmit functions for ISD::SUB.
8873
8874Register fastEmit_ISD_SUB_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8875 if (RetVT.SimpleTy != MVT::i32)
8876 return Register();
8877 if ((!Subtarget->isThumb())) {
8878 return fastEmitInst_ri(MachineInstOpcode: ARM::SUBri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8879 }
8880 return Register();
8881}
8882
8883Register fastEmit_ISD_SUB_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8884 switch (VT.SimpleTy) {
8885 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8886 default: return Register();
8887 }
8888}
8889
8890// FastEmit functions for ISD::XOR.
8891
8892Register fastEmit_ISD_XOR_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8893 if (RetVT.SimpleTy != MVT::i32)
8894 return Register();
8895 if ((!Subtarget->isThumb())) {
8896 return fastEmitInst_ri(MachineInstOpcode: ARM::EORri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8897 }
8898 return Register();
8899}
8900
8901Register fastEmit_ISD_XOR_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8902 switch (VT.SimpleTy) {
8903 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8904 default: return Register();
8905 }
8906}
8907
8908// Top-level FastEmit function.
8909
8910Register fastEmit_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8911 switch (Opcode) {
8912 case ARMISD::CMN: return fastEmit_ARMISD_CMN_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8913 case ARMISD::CMP: return fastEmit_ARMISD_CMP_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8914 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8915 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8916 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8917 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8918 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8919 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8920 default: return Register();
8921 }
8922}
8923
8924// FastEmit functions for ARMISD::VSHLIMM.
8925
8926Register fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
8927 if (RetVT.SimpleTy != MVT::v16i8)
8928 return Register();
8929 if ((Subtarget->hasMVEIntegerOps())) {
8930 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHL_immi8, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8931 }
8932 return Register();
8933}
8934
8935Register fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8936 switch (VT.SimpleTy) {
8937 case MVT::v16i8: return fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, imm1);
8938 default: return Register();
8939 }
8940}
8941
8942// FastEmit functions for ARMISD::VSHRsIMM.
8943
8944Register fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
8945 if (RetVT.SimpleTy != MVT::v16i8)
8946 return Register();
8947 if ((Subtarget->hasMVEIntegerOps())) {
8948 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_imms8, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8949 }
8950 return Register();
8951}
8952
8953Register fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8954 switch (VT.SimpleTy) {
8955 case MVT::v16i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, imm1);
8956 default: return Register();
8957 }
8958}
8959
8960// FastEmit functions for ARMISD::VSHRuIMM.
8961
8962Register fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
8963 if (RetVT.SimpleTy != MVT::v16i8)
8964 return Register();
8965 if ((Subtarget->hasMVEIntegerOps())) {
8966 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_immu8, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8967 }
8968 return Register();
8969}
8970
8971Register fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8972 switch (VT.SimpleTy) {
8973 case MVT::v16i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, imm1);
8974 default: return Register();
8975 }
8976}
8977
8978// FastEmit functions for ISD::ADD.
8979
8980Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
8981 if (RetVT.SimpleTy != MVT::i32)
8982 return Register();
8983 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
8984 return fastEmitInst_ri(MachineInstOpcode: ARM::tADDi3, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
8985 }
8986 return Register();
8987}
8988
8989Register fastEmit_ISD_ADD_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8990 switch (VT.SimpleTy) {
8991 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_7(RetVT, Op0, imm1);
8992 default: return Register();
8993 }
8994}
8995
8996// Top-level FastEmit function.
8997
8998Register fastEmit_ri_Predicate_imm0_7(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8999 switch (Opcode) {
9000 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
9001 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
9002 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
9003 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
9004 default: return Register();
9005 }
9006}
9007
9008// FastEmit functions for ISD::ADD.
9009
9010Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_255_expr(MVT RetVT, Register Op0, uint64_t imm1) {
9011 if (RetVT.SimpleTy != MVT::i32)
9012 return Register();
9013 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
9014 return fastEmitInst_ri(MachineInstOpcode: ARM::tADDi8, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
9015 }
9016 return Register();
9017}
9018
9019Register fastEmit_ISD_ADD_ri_Predicate_imm0_255_expr(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9020 switch (VT.SimpleTy) {
9021 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_255_expr(RetVT, Op0, imm1);
9022 default: return Register();
9023 }
9024}
9025
9026// Top-level FastEmit function.
9027
9028Register fastEmit_ri_Predicate_imm0_255_expr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9029 switch (Opcode) {
9030 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm0_255_expr(VT, RetVT, Op0, imm1);
9031 default: return Register();
9032 }
9033}
9034
9035// FastEmit functions for ARMISD::CMP.
9036
9037Register fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_imm0_255(MVT RetVT, Register Op0, uint64_t imm1) {
9038 if (RetVT.SimpleTy != MVT::i32)
9039 return Register();
9040 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
9041 return fastEmitInst_ri(MachineInstOpcode: ARM::tCMPi8, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
9042 }
9043 return Register();
9044}
9045
9046Register fastEmit_ARMISD_CMP_ri_Predicate_imm0_255(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9047 switch (VT.SimpleTy) {
9048 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_imm0_255(RetVT, Op0, imm1);
9049 default: return Register();
9050 }
9051}
9052
9053// FastEmit functions for ARMISD::CMPZ.
9054
9055Register fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_imm0_255(MVT RetVT, Register Op0, uint64_t imm1) {
9056 if (RetVT.SimpleTy != MVT::i32)
9057 return Register();
9058 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
9059 return fastEmitInst_ri(MachineInstOpcode: ARM::tCMPi8, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
9060 }
9061 return Register();
9062}
9063
9064Register fastEmit_ARMISD_CMPZ_ri_Predicate_imm0_255(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9065 switch (VT.SimpleTy) {
9066 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_imm0_255(RetVT, Op0, imm1);
9067 default: return Register();
9068 }
9069}
9070
9071// Top-level FastEmit function.
9072
9073Register fastEmit_ri_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9074 switch (Opcode) {
9075 case ARMISD::CMP: return fastEmit_ARMISD_CMP_ri_Predicate_imm0_255(VT, RetVT, Op0, imm1);
9076 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_ri_Predicate_imm0_255(VT, RetVT, Op0, imm1);
9077 default: return Register();
9078 }
9079}
9080
9081// FastEmit functions for ARMISD::CMP.
9082
9083Register fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
9084 if (RetVT.SimpleTy != MVT::i32)
9085 return Register();
9086 if ((Subtarget->isThumb2())) {
9087 return fastEmitInst_ri(MachineInstOpcode: ARM::t2CMPri, RC: &ARM::GPRnopcRegClass, Op0, Imm: imm1);
9088 }
9089 return Register();
9090}
9091
9092Register fastEmit_ARMISD_CMP_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9093 switch (VT.SimpleTy) {
9094 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
9095 default: return Register();
9096 }
9097}
9098
9099// FastEmit functions for ARMISD::CMPZ.
9100
9101Register fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
9102 if (RetVT.SimpleTy != MVT::i32)
9103 return Register();
9104 if ((Subtarget->isThumb2())) {
9105 return fastEmitInst_ri(MachineInstOpcode: ARM::t2CMPri, RC: &ARM::GPRnopcRegClass, Op0, Imm: imm1);
9106 }
9107 return Register();
9108}
9109
9110Register fastEmit_ARMISD_CMPZ_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9111 switch (VT.SimpleTy) {
9112 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
9113 default: return Register();
9114 }
9115}
9116
9117// FastEmit functions for ISD::ADD.
9118
9119Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
9120 if (RetVT.SimpleTy != MVT::i32)
9121 return Register();
9122 if ((Subtarget->isThumb2())) {
9123 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ADDri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
9124 }
9125 return Register();
9126}
9127
9128Register fastEmit_ISD_ADD_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9129 switch (VT.SimpleTy) {
9130 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
9131 default: return Register();
9132 }
9133}
9134
9135// FastEmit functions for ISD::AND.
9136
9137Register fastEmit_ISD_AND_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
9138 if (RetVT.SimpleTy != MVT::i32)
9139 return Register();
9140 if ((Subtarget->isThumb2())) {
9141 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ANDri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
9142 }
9143 return Register();
9144}
9145
9146Register fastEmit_ISD_AND_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9147 switch (VT.SimpleTy) {
9148 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
9149 default: return Register();
9150 }
9151}
9152
9153// FastEmit functions for ISD::OR.
9154
9155Register fastEmit_ISD_OR_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
9156 if (RetVT.SimpleTy != MVT::i32)
9157 return Register();
9158 if ((Subtarget->isThumb2())) {
9159 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ORRri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
9160 }
9161 return Register();
9162}
9163
9164Register fastEmit_ISD_OR_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9165 switch (VT.SimpleTy) {
9166 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
9167 default: return Register();
9168 }
9169}
9170
9171// FastEmit functions for ISD::SUB.
9172
9173Register fastEmit_ISD_SUB_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
9174 if (RetVT.SimpleTy != MVT::i32)
9175 return Register();
9176 if ((Subtarget->isThumb2())) {
9177 return fastEmitInst_ri(MachineInstOpcode: ARM::t2SUBri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
9178 }
9179 return Register();
9180}
9181
9182Register fastEmit_ISD_SUB_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9183 switch (VT.SimpleTy) {
9184 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
9185 default: return Register();
9186 }
9187}
9188
9189// FastEmit functions for ISD::XOR.
9190
9191Register fastEmit_ISD_XOR_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
9192 if (RetVT.SimpleTy != MVT::i32)
9193 return Register();
9194 if ((Subtarget->isThumb2())) {
9195 return fastEmitInst_ri(MachineInstOpcode: ARM::t2EORri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
9196 }
9197 return Register();
9198}
9199
9200Register fastEmit_ISD_XOR_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9201 switch (VT.SimpleTy) {
9202 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
9203 default: return Register();
9204 }
9205}
9206
9207// Top-level FastEmit function.
9208
9209Register fastEmit_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9210 switch (Opcode) {
9211 case ARMISD::CMP: return fastEmit_ARMISD_CMP_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
9212 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
9213 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
9214 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
9215 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
9216 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
9217 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
9218 default: return Register();
9219 }
9220}
9221
9222// FastEmit functions for ISD::ADD.
9223
9224Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_4095(MVT RetVT, Register Op0, uint64_t imm1) {
9225 if (RetVT.SimpleTy != MVT::i32)
9226 return Register();
9227 if ((Subtarget->isThumb2())) {
9228 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ADDri12, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
9229 }
9230 return Register();
9231}
9232
9233Register fastEmit_ISD_ADD_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9234 switch (VT.SimpleTy) {
9235 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_4095(RetVT, Op0, imm1);
9236 default: return Register();
9237 }
9238}
9239
9240// FastEmit functions for ISD::SUB.
9241
9242Register fastEmit_ISD_SUB_MVT_i32_ri_Predicate_imm0_4095(MVT RetVT, Register Op0, uint64_t imm1) {
9243 if (RetVT.SimpleTy != MVT::i32)
9244 return Register();
9245 if ((Subtarget->isThumb2())) {
9246 return fastEmitInst_ri(MachineInstOpcode: ARM::t2SUBri12, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
9247 }
9248 return Register();
9249}
9250
9251Register fastEmit_ISD_SUB_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9252 switch (VT.SimpleTy) {
9253 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_imm0_4095(RetVT, Op0, imm1);
9254 default: return Register();
9255 }
9256}
9257
9258// Top-level FastEmit function.
9259
9260Register fastEmit_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9261 switch (Opcode) {
9262 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm0_4095(VT, RetVT, Op0, imm1);
9263 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_imm0_4095(VT, RetVT, Op0, imm1);
9264 default: return Register();
9265 }
9266}
9267
9268// FastEmit functions for ISD::ROTR.
9269
9270Register fastEmit_ISD_ROTR_MVT_i32_ri_Predicate_imm1_31(MVT RetVT, Register Op0, uint64_t imm1) {
9271 if (RetVT.SimpleTy != MVT::i32)
9272 return Register();
9273 if ((Subtarget->isThumb2())) {
9274 return fastEmitInst_ri(MachineInstOpcode: ARM::t2RORri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
9275 }
9276 return Register();
9277}
9278
9279Register fastEmit_ISD_ROTR_ri_Predicate_imm1_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9280 switch (VT.SimpleTy) {
9281 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_ri_Predicate_imm1_31(RetVT, Op0, imm1);
9282 default: return Register();
9283 }
9284}
9285
9286// FastEmit functions for ISD::SHL.
9287
9288Register fastEmit_ISD_SHL_MVT_i32_ri_Predicate_imm1_31(MVT RetVT, Register Op0, uint64_t imm1) {
9289 if (RetVT.SimpleTy != MVT::i32)
9290 return Register();
9291 if ((Subtarget->isThumb2())) {
9292 return fastEmitInst_ri(MachineInstOpcode: ARM::t2LSLri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
9293 }
9294 return Register();
9295}
9296
9297Register fastEmit_ISD_SHL_ri_Predicate_imm1_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9298 switch (VT.SimpleTy) {
9299 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri_Predicate_imm1_31(RetVT, Op0, imm1);
9300 default: return Register();
9301 }
9302}
9303
9304// Top-level FastEmit function.
9305
9306Register fastEmit_ri_Predicate_imm1_31(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9307 switch (Opcode) {
9308 case ISD::ROTR: return fastEmit_ISD_ROTR_ri_Predicate_imm1_31(VT, RetVT, Op0, imm1);
9309 case ISD::SHL: return fastEmit_ISD_SHL_ri_Predicate_imm1_31(VT, RetVT, Op0, imm1);
9310 default: return Register();
9311 }
9312}
9313
9314// FastEmit functions for ARMISD::VQRSHRNsIMM.
9315
9316Register fastEmit_ARMISD_VQRSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
9317 if (RetVT.SimpleTy != MVT::v8i8)
9318 return Register();
9319 if ((Subtarget->hasNEON())) {
9320 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9321 }
9322 return Register();
9323}
9324
9325Register fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9326 switch (VT.SimpleTy) {
9327 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
9328 default: return Register();
9329 }
9330}
9331
9332// FastEmit functions for ARMISD::VQRSHRNsuIMM.
9333
9334Register fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
9335 if (RetVT.SimpleTy != MVT::v8i8)
9336 return Register();
9337 if ((Subtarget->hasNEON())) {
9338 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRUNv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9339 }
9340 return Register();
9341}
9342
9343Register fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9344 switch (VT.SimpleTy) {
9345 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
9346 default: return Register();
9347 }
9348}
9349
9350// FastEmit functions for ARMISD::VQRSHRNuIMM.
9351
9352Register fastEmit_ARMISD_VQRSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
9353 if (RetVT.SimpleTy != MVT::v8i8)
9354 return Register();
9355 if ((Subtarget->hasNEON())) {
9356 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9357 }
9358 return Register();
9359}
9360
9361Register fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9362 switch (VT.SimpleTy) {
9363 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
9364 default: return Register();
9365 }
9366}
9367
9368// FastEmit functions for ARMISD::VQSHRNsIMM.
9369
9370Register fastEmit_ARMISD_VQSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
9371 if (RetVT.SimpleTy != MVT::v8i8)
9372 return Register();
9373 if ((Subtarget->hasNEON())) {
9374 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9375 }
9376 return Register();
9377}
9378
9379Register fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9380 switch (VT.SimpleTy) {
9381 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
9382 default: return Register();
9383 }
9384}
9385
9386// FastEmit functions for ARMISD::VQSHRNsuIMM.
9387
9388Register fastEmit_ARMISD_VQSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
9389 if (RetVT.SimpleTy != MVT::v8i8)
9390 return Register();
9391 if ((Subtarget->hasNEON())) {
9392 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRUNv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9393 }
9394 return Register();
9395}
9396
9397Register fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9398 switch (VT.SimpleTy) {
9399 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
9400 default: return Register();
9401 }
9402}
9403
9404// FastEmit functions for ARMISD::VQSHRNuIMM.
9405
9406Register fastEmit_ARMISD_VQSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
9407 if (RetVT.SimpleTy != MVT::v8i8)
9408 return Register();
9409 if ((Subtarget->hasNEON())) {
9410 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9411 }
9412 return Register();
9413}
9414
9415Register fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9416 switch (VT.SimpleTy) {
9417 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
9418 default: return Register();
9419 }
9420}
9421
9422// FastEmit functions for ARMISD::VRSHRNIMM.
9423
9424Register fastEmit_ARMISD_VRSHRNIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
9425 if (RetVT.SimpleTy != MVT::v8i8)
9426 return Register();
9427 if ((Subtarget->hasNEON())) {
9428 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRNv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9429 }
9430 return Register();
9431}
9432
9433Register fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9434 switch (VT.SimpleTy) {
9435 case MVT::v8i16: return fastEmit_ARMISD_VRSHRNIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
9436 default: return Register();
9437 }
9438}
9439
9440// Top-level FastEmit function.
9441
9442Register fastEmit_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9443 switch (Opcode) {
9444 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
9445 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
9446 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
9447 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
9448 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
9449 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
9450 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
9451 default: return Register();
9452 }
9453}
9454
9455// FastEmit functions for ARMISD::VQRSHRNsIMM.
9456
9457Register fastEmit_ARMISD_VQRSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
9458 if (RetVT.SimpleTy != MVT::v4i16)
9459 return Register();
9460 if ((Subtarget->hasNEON())) {
9461 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9462 }
9463 return Register();
9464}
9465
9466Register fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9467 switch (VT.SimpleTy) {
9468 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
9469 default: return Register();
9470 }
9471}
9472
9473// FastEmit functions for ARMISD::VQRSHRNsuIMM.
9474
9475Register fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
9476 if (RetVT.SimpleTy != MVT::v4i16)
9477 return Register();
9478 if ((Subtarget->hasNEON())) {
9479 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRUNv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9480 }
9481 return Register();
9482}
9483
9484Register fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9485 switch (VT.SimpleTy) {
9486 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
9487 default: return Register();
9488 }
9489}
9490
9491// FastEmit functions for ARMISD::VQRSHRNuIMM.
9492
9493Register fastEmit_ARMISD_VQRSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
9494 if (RetVT.SimpleTy != MVT::v4i16)
9495 return Register();
9496 if ((Subtarget->hasNEON())) {
9497 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9498 }
9499 return Register();
9500}
9501
9502Register fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9503 switch (VT.SimpleTy) {
9504 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
9505 default: return Register();
9506 }
9507}
9508
9509// FastEmit functions for ARMISD::VQSHRNsIMM.
9510
9511Register fastEmit_ARMISD_VQSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
9512 if (RetVT.SimpleTy != MVT::v4i16)
9513 return Register();
9514 if ((Subtarget->hasNEON())) {
9515 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9516 }
9517 return Register();
9518}
9519
9520Register fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9521 switch (VT.SimpleTy) {
9522 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
9523 default: return Register();
9524 }
9525}
9526
9527// FastEmit functions for ARMISD::VQSHRNsuIMM.
9528
9529Register fastEmit_ARMISD_VQSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
9530 if (RetVT.SimpleTy != MVT::v4i16)
9531 return Register();
9532 if ((Subtarget->hasNEON())) {
9533 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRUNv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9534 }
9535 return Register();
9536}
9537
9538Register fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9539 switch (VT.SimpleTy) {
9540 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
9541 default: return Register();
9542 }
9543}
9544
9545// FastEmit functions for ARMISD::VQSHRNuIMM.
9546
9547Register fastEmit_ARMISD_VQSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
9548 if (RetVT.SimpleTy != MVT::v4i16)
9549 return Register();
9550 if ((Subtarget->hasNEON())) {
9551 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9552 }
9553 return Register();
9554}
9555
9556Register fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9557 switch (VT.SimpleTy) {
9558 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
9559 default: return Register();
9560 }
9561}
9562
9563// FastEmit functions for ARMISD::VRSHRNIMM.
9564
9565Register fastEmit_ARMISD_VRSHRNIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
9566 if (RetVT.SimpleTy != MVT::v4i16)
9567 return Register();
9568 if ((Subtarget->hasNEON())) {
9569 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRNv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9570 }
9571 return Register();
9572}
9573
9574Register fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9575 switch (VT.SimpleTy) {
9576 case MVT::v4i32: return fastEmit_ARMISD_VRSHRNIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
9577 default: return Register();
9578 }
9579}
9580
9581// Top-level FastEmit function.
9582
9583Register fastEmit_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9584 switch (Opcode) {
9585 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9586 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9587 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9588 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9589 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9590 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9591 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9592 default: return Register();
9593 }
9594}
9595
9596// FastEmit functions for ARMISD::VQRSHRNsIMM.
9597
9598Register fastEmit_ARMISD_VQRSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9599 if (RetVT.SimpleTy != MVT::v2i32)
9600 return Register();
9601 if ((Subtarget->hasNEON())) {
9602 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9603 }
9604 return Register();
9605}
9606
9607Register fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9608 switch (VT.SimpleTy) {
9609 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9610 default: return Register();
9611 }
9612}
9613
9614// FastEmit functions for ARMISD::VQRSHRNsuIMM.
9615
9616Register fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9617 if (RetVT.SimpleTy != MVT::v2i32)
9618 return Register();
9619 if ((Subtarget->hasNEON())) {
9620 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRUNv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9621 }
9622 return Register();
9623}
9624
9625Register fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9626 switch (VT.SimpleTy) {
9627 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9628 default: return Register();
9629 }
9630}
9631
9632// FastEmit functions for ARMISD::VQRSHRNuIMM.
9633
9634Register fastEmit_ARMISD_VQRSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9635 if (RetVT.SimpleTy != MVT::v2i32)
9636 return Register();
9637 if ((Subtarget->hasNEON())) {
9638 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9639 }
9640 return Register();
9641}
9642
9643Register fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9644 switch (VT.SimpleTy) {
9645 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9646 default: return Register();
9647 }
9648}
9649
9650// FastEmit functions for ARMISD::VQSHRNsIMM.
9651
9652Register fastEmit_ARMISD_VQSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9653 if (RetVT.SimpleTy != MVT::v2i32)
9654 return Register();
9655 if ((Subtarget->hasNEON())) {
9656 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9657 }
9658 return Register();
9659}
9660
9661Register fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9662 switch (VT.SimpleTy) {
9663 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9664 default: return Register();
9665 }
9666}
9667
9668// FastEmit functions for ARMISD::VQSHRNsuIMM.
9669
9670Register fastEmit_ARMISD_VQSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9671 if (RetVT.SimpleTy != MVT::v2i32)
9672 return Register();
9673 if ((Subtarget->hasNEON())) {
9674 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRUNv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9675 }
9676 return Register();
9677}
9678
9679Register fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9680 switch (VT.SimpleTy) {
9681 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9682 default: return Register();
9683 }
9684}
9685
9686// FastEmit functions for ARMISD::VQSHRNuIMM.
9687
9688Register fastEmit_ARMISD_VQSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9689 if (RetVT.SimpleTy != MVT::v2i32)
9690 return Register();
9691 if ((Subtarget->hasNEON())) {
9692 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9693 }
9694 return Register();
9695}
9696
9697Register fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9698 switch (VT.SimpleTy) {
9699 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9700 default: return Register();
9701 }
9702}
9703
9704// FastEmit functions for ARMISD::VRSHRNIMM.
9705
9706Register fastEmit_ARMISD_VRSHRNIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9707 if (RetVT.SimpleTy != MVT::v2i32)
9708 return Register();
9709 if ((Subtarget->hasNEON())) {
9710 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRNv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9711 }
9712 return Register();
9713}
9714
9715Register fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9716 switch (VT.SimpleTy) {
9717 case MVT::v2i64: return fastEmit_ARMISD_VRSHRNIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9718 default: return Register();
9719 }
9720}
9721
9722// Top-level FastEmit function.
9723
9724Register fastEmit_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9725 switch (Opcode) {
9726 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9727 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9728 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9729 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9730 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9731 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9732 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9733 default: return Register();
9734 }
9735}
9736
9737// FastEmit functions for ARMISD::VDUPLANE.
9738
9739Register fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri_Predicate_VectorIndex32(MVT RetVT, Register Op0, uint64_t imm1) {
9740 if (RetVT.SimpleTy != MVT::v16i8)
9741 return Register();
9742 if ((Subtarget->hasNEON())) {
9743 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN8q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
9744 }
9745 return Register();
9746}
9747
9748Register fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri_Predicate_VectorIndex32(MVT RetVT, Register Op0, uint64_t imm1) {
9749 if (RetVT.SimpleTy != MVT::v8i16)
9750 return Register();
9751 if ((Subtarget->hasNEON())) {
9752 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
9753 }
9754 return Register();
9755}
9756
9757Register fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri_Predicate_VectorIndex32(MVT RetVT, Register Op0, uint64_t imm1) {
9758 if (RetVT.SimpleTy != MVT::v4i32)
9759 return Register();
9760 if ((Subtarget->hasNEON())) {
9761 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
9762 }
9763 return Register();
9764}
9765
9766Register fastEmit_ARMISD_VDUPLANE_ri_Predicate_VectorIndex32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9767 switch (VT.SimpleTy) {
9768 case MVT::v8i8: return fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri_Predicate_VectorIndex32(RetVT, Op0, imm1);
9769 case MVT::v4i16: return fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri_Predicate_VectorIndex32(RetVT, Op0, imm1);
9770 case MVT::v2i32: return fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri_Predicate_VectorIndex32(RetVT, Op0, imm1);
9771 default: return Register();
9772 }
9773}
9774
9775// Top-level FastEmit function.
9776
9777Register fastEmit_ri_Predicate_VectorIndex32(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9778 switch (Opcode) {
9779 case ARMISD::VDUPLANE: return fastEmit_ARMISD_VDUPLANE_ri_Predicate_VectorIndex32(VT, RetVT, Op0, imm1);
9780 default: return Register();
9781 }
9782}
9783
9784// FastEmit functions for ARMISD::VSHLIMM.
9785
9786Register fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, Register Op0, uint64_t imm1) {
9787 if (RetVT.SimpleTy != MVT::v4i32)
9788 return Register();
9789 if ((Subtarget->hasMVEIntegerOps())) {
9790 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHL_immi32, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9791 }
9792 return Register();
9793}
9794
9795Register fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9796 switch (VT.SimpleTy) {
9797 case MVT::v4i32: return fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, imm1);
9798 default: return Register();
9799 }
9800}
9801
9802// FastEmit functions for ARMISD::VSHRsIMM.
9803
9804Register fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, Register Op0, uint64_t imm1) {
9805 if (RetVT.SimpleTy != MVT::v4i32)
9806 return Register();
9807 if ((Subtarget->hasMVEIntegerOps())) {
9808 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_imms32, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9809 }
9810 return Register();
9811}
9812
9813Register fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9814 switch (VT.SimpleTy) {
9815 case MVT::v4i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, imm1);
9816 default: return Register();
9817 }
9818}
9819
9820// FastEmit functions for ARMISD::VSHRuIMM.
9821
9822Register fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, Register Op0, uint64_t imm1) {
9823 if (RetVT.SimpleTy != MVT::v4i32)
9824 return Register();
9825 if ((Subtarget->hasMVEIntegerOps())) {
9826 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_immu32, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9827 }
9828 return Register();
9829}
9830
9831Register fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9832 switch (VT.SimpleTy) {
9833 case MVT::v4i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, imm1);
9834 default: return Register();
9835 }
9836}
9837
9838// Top-level FastEmit function.
9839
9840Register fastEmit_ri_Predicate_imm0_31(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9841 switch (Opcode) {
9842 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, imm1);
9843 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, imm1);
9844 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, imm1);
9845 default: return Register();
9846 }
9847}
9848
9849// FastEmit functions for ARMISD::VSHLIMM.
9850
9851Register fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, Register Op0, uint64_t imm1) {
9852 if (RetVT.SimpleTy != MVT::v8i16)
9853 return Register();
9854 if ((Subtarget->hasMVEIntegerOps())) {
9855 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHL_immi16, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9856 }
9857 return Register();
9858}
9859
9860Register fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9861 switch (VT.SimpleTy) {
9862 case MVT::v8i16: return fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, imm1);
9863 default: return Register();
9864 }
9865}
9866
9867// FastEmit functions for ARMISD::VSHRsIMM.
9868
9869Register fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, Register Op0, uint64_t imm1) {
9870 if (RetVT.SimpleTy != MVT::v8i16)
9871 return Register();
9872 if ((Subtarget->hasMVEIntegerOps())) {
9873 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_imms16, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9874 }
9875 return Register();
9876}
9877
9878Register fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9879 switch (VT.SimpleTy) {
9880 case MVT::v8i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, imm1);
9881 default: return Register();
9882 }
9883}
9884
9885// FastEmit functions for ARMISD::VSHRuIMM.
9886
9887Register fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, Register Op0, uint64_t imm1) {
9888 if (RetVT.SimpleTy != MVT::v8i16)
9889 return Register();
9890 if ((Subtarget->hasMVEIntegerOps())) {
9891 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_immu16, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9892 }
9893 return Register();
9894}
9895
9896Register fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9897 switch (VT.SimpleTy) {
9898 case MVT::v8i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, imm1);
9899 default: return Register();
9900 }
9901}
9902
9903// Top-level FastEmit function.
9904
9905Register fastEmit_ri_Predicate_imm0_15(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9906 switch (Opcode) {
9907 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, imm1);
9908 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, imm1);
9909 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, imm1);
9910 default: return Register();
9911 }
9912}
9913
9914// FastEmit functions for ISD::Constant.
9915
9916Register fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
9917 if (RetVT.SimpleTy != MVT::i32)
9918 return Register();
9919 if ((Subtarget->isThumb()) && (Subtarget->useMovt())) {
9920 return fastEmitInst_i(MachineInstOpcode: ARM::t2MOVi32imm, RC: &ARM::rGPRRegClass, Imm: imm0);
9921 }
9922 if ((!Subtarget->useMovt()) && (Subtarget->genExecuteOnly()) && (Subtarget->isThumb1Only())) {
9923 return fastEmitInst_i(MachineInstOpcode: ARM::tMOVi32imm, RC: &ARM::rGPRRegClass, Imm: imm0);
9924 }
9925 return Register();
9926}
9927
9928Register fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
9929 switch (VT.SimpleTy) {
9930 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
9931 default: return Register();
9932 }
9933}
9934
9935// Top-level FastEmit function.
9936
9937Register fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
9938 switch (Opcode) {
9939 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
9940 default: return Register();
9941 }
9942}
9943
9944