1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the ARM target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_mod_imm(int64_t Imm) {
12
13 return ARM_AM::getSOImmVal(Arg: Imm) != -1;
14
15}
16static bool Predicate_imm0_7(int64_t Imm) {
17
18 return Imm >= 0 && Imm < 8;
19
20}
21static bool Predicate_imm0_255_expr(int64_t Imm) {
22 return Imm >= 0 && Imm < 256;
23}
24static bool Predicate_imm_sr(int64_t Imm) {
25
26 return Imm > 0 && Imm <= 32;
27
28}
29static bool Predicate_imm0_255(int64_t Imm) {
30 return Imm >= 0 && Imm < 256;
31}
32static bool Predicate_t2_so_imm(int64_t Imm) {
33
34 return ARM_AM::getT2SOImmVal(Arg: Imm) != -1;
35
36}
37static bool Predicate_imm0_4095(int64_t Imm) {
38
39 return Imm >= 0 && Imm < 4096;
40
41}
42static bool Predicate_imm1_31(int64_t Imm) {
43 return Imm > 0 && Imm < 32;
44}
45static bool Predicate_shr_imm8(int64_t Imm) {
46 return Imm > 0 && Imm <= 8;
47}
48static bool Predicate_shr_imm16(int64_t Imm) {
49 return Imm > 0 && Imm <= 16;
50}
51static bool Predicate_shr_imm32(int64_t Imm) {
52 return Imm > 0 && Imm <= 32;
53}
54static bool Predicate_VectorIndex32(int64_t Imm) {
55
56 return ((uint64_t)Imm) < 2;
57
58}
59static bool Predicate_imm0_31(int64_t Imm) {
60
61 return Imm >= 0 && Imm < 32;
62
63}
64static bool Predicate_mod_imm_not(int64_t Imm) {
65
66 return ARM_AM::getSOImmVal(Arg: ~(uint32_t)Imm) != -1;
67
68}
69static bool Predicate_imm0_65535(int64_t Imm) {
70
71 return Imm >= 0 && Imm < 65536;
72
73}
74static bool Predicate_t2_so_imm_neg(int64_t Imm) {
75
76 return Imm && ARM_AM::getT2SOImmVal(Arg: -(uint32_t)Imm) != -1;
77
78}
79static bool Predicate_imm0_15(int64_t Imm) {
80
81 return Imm >= 0 && Imm < 16;
82
83}
84
85
86// FastEmit functions for ISD::GET_FPENV.
87
88Register fastEmit_ISD_GET_FPENV_MVT_i32_(MVT RetVT) {
89 if (RetVT.SimpleTy != MVT::i32)
90 return Register();
91 return fastEmitInst_(MachineInstOpcode: ARM::VMRS, RC: &ARM::GPRnopcRegClass);
92}
93
94Register fastEmit_ISD_GET_FPENV_(MVT VT, MVT RetVT) {
95 switch (VT.SimpleTy) {
96 case MVT::i32: return fastEmit_ISD_GET_FPENV_MVT_i32_(RetVT);
97 default: return Register();
98 }
99}
100
101// FastEmit functions for ISD::GET_FPMODE.
102
103Register fastEmit_ISD_GET_FPMODE_MVT_i32_(MVT RetVT) {
104 if (RetVT.SimpleTy != MVT::i32)
105 return Register();
106 return fastEmitInst_(MachineInstOpcode: ARM::VMRS, RC: &ARM::GPRnopcRegClass);
107}
108
109Register fastEmit_ISD_GET_FPMODE_(MVT VT, MVT RetVT) {
110 switch (VT.SimpleTy) {
111 case MVT::i32: return fastEmit_ISD_GET_FPMODE_MVT_i32_(RetVT);
112 default: return Register();
113 }
114}
115
116// Top-level FastEmit function.
117
118Register fastEmit_(MVT VT, MVT RetVT, unsigned Opcode) override {
119 switch (Opcode) {
120 case ISD::GET_FPENV: return fastEmit_ISD_GET_FPENV_(VT, RetVT);
121 case ISD::GET_FPMODE: return fastEmit_ISD_GET_FPMODE_(VT, RetVT);
122 default: return Register();
123 }
124}
125
126// FastEmit functions for ARMISD::CALL.
127
128Register fastEmit_ARMISD_CALL_MVT_i32_r(MVT RetVT, Register Op0) {
129 if (RetVT.SimpleTy != MVT::isVoid)
130 return Register();
131 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
132 return fastEmitInst_r(MachineInstOpcode: ARM::BLX_noip, RC: &ARM::GPRnoipRegClass, Op0);
133 }
134 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( !MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
135 return fastEmitInst_r(MachineInstOpcode: ARM::BLX, RC: &ARM::GPRRegClass, Op0);
136 }
137 return Register();
138}
139
140Register fastEmit_ARMISD_CALL_r(MVT VT, MVT RetVT, Register Op0) {
141 switch (VT.SimpleTy) {
142 case MVT::i32: return fastEmit_ARMISD_CALL_MVT_i32_r(RetVT, Op0);
143 default: return Register();
144 }
145}
146
147// FastEmit functions for ARMISD::CALL_NOLINK.
148
149Register fastEmit_ARMISD_CALL_NOLINK_MVT_i32_r(MVT RetVT, Register Op0) {
150 if (RetVT.SimpleTy != MVT::isVoid)
151 return Register();
152 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
153 return fastEmitInst_r(MachineInstOpcode: ARM::tBX_CALL, RC: &ARM::tGPRRegClass, Op0);
154 }
155 if ((!Subtarget->isThumb()) && (!Subtarget->hasV4TOps())) {
156 return fastEmitInst_r(MachineInstOpcode: ARM::BMOVPCRX_CALL, RC: &ARM::tGPRRegClass, Op0);
157 }
158 if ((Subtarget->hasV4TOps()) && (!Subtarget->isThumb())) {
159 return fastEmitInst_r(MachineInstOpcode: ARM::BX_CALL, RC: &ARM::tGPRRegClass, Op0);
160 }
161 return Register();
162}
163
164Register fastEmit_ARMISD_CALL_NOLINK_r(MVT VT, MVT RetVT, Register Op0) {
165 switch (VT.SimpleTy) {
166 case MVT::i32: return fastEmit_ARMISD_CALL_NOLINK_MVT_i32_r(RetVT, Op0);
167 default: return Register();
168 }
169}
170
171// FastEmit functions for ARMISD::CALL_PRED.
172
173Register fastEmit_ARMISD_CALL_PRED_MVT_i32_r(MVT RetVT, Register Op0) {
174 if (RetVT.SimpleTy != MVT::isVoid)
175 return Register();
176 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
177 return fastEmitInst_r(MachineInstOpcode: ARM::BLX_pred_noip, RC: &ARM::GPRnoipRegClass, Op0);
178 }
179 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( !MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
180 return fastEmitInst_r(MachineInstOpcode: ARM::BLX_pred, RC: &ARM::GPRRegClass, Op0);
181 }
182 return Register();
183}
184
185Register fastEmit_ARMISD_CALL_PRED_r(MVT VT, MVT RetVT, Register Op0) {
186 switch (VT.SimpleTy) {
187 case MVT::i32: return fastEmit_ARMISD_CALL_PRED_MVT_i32_r(RetVT, Op0);
188 default: return Register();
189 }
190}
191
192// FastEmit functions for ARMISD::CMPFPEw0.
193
194Register fastEmit_ARMISD_CMPFPEw0_MVT_f16_r(MVT RetVT, Register Op0) {
195 if (RetVT.SimpleTy != MVT::i32)
196 return Register();
197 if ((Subtarget->hasFullFP16())) {
198 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPEZH, RC: &ARM::HPRRegClass, Op0);
199 }
200 return Register();
201}
202
203Register fastEmit_ARMISD_CMPFPEw0_MVT_f32_r(MVT RetVT, Register Op0) {
204 if (RetVT.SimpleTy != MVT::i32)
205 return Register();
206 if ((Subtarget->hasVFP2Base())) {
207 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPEZS, RC: &ARM::SPRRegClass, Op0);
208 }
209 return Register();
210}
211
212Register fastEmit_ARMISD_CMPFPEw0_MVT_f64_r(MVT RetVT, Register Op0) {
213 if (RetVT.SimpleTy != MVT::i32)
214 return Register();
215 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
216 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPEZD, RC: &ARM::DPRRegClass, Op0);
217 }
218 return Register();
219}
220
221Register fastEmit_ARMISD_CMPFPEw0_r(MVT VT, MVT RetVT, Register Op0) {
222 switch (VT.SimpleTy) {
223 case MVT::f16: return fastEmit_ARMISD_CMPFPEw0_MVT_f16_r(RetVT, Op0);
224 case MVT::f32: return fastEmit_ARMISD_CMPFPEw0_MVT_f32_r(RetVT, Op0);
225 case MVT::f64: return fastEmit_ARMISD_CMPFPEw0_MVT_f64_r(RetVT, Op0);
226 default: return Register();
227 }
228}
229
230// FastEmit functions for ARMISD::CMPFPw0.
231
232Register fastEmit_ARMISD_CMPFPw0_MVT_f16_r(MVT RetVT, Register Op0) {
233 if (RetVT.SimpleTy != MVT::i32)
234 return Register();
235 if ((Subtarget->hasFullFP16())) {
236 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPZH, RC: &ARM::HPRRegClass, Op0);
237 }
238 return Register();
239}
240
241Register fastEmit_ARMISD_CMPFPw0_MVT_f32_r(MVT RetVT, Register Op0) {
242 if (RetVT.SimpleTy != MVT::i32)
243 return Register();
244 if ((Subtarget->hasVFP2Base())) {
245 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPZS, RC: &ARM::SPRRegClass, Op0);
246 }
247 return Register();
248}
249
250Register fastEmit_ARMISD_CMPFPw0_MVT_f64_r(MVT RetVT, Register Op0) {
251 if (RetVT.SimpleTy != MVT::i32)
252 return Register();
253 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
254 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPZD, RC: &ARM::DPRRegClass, Op0);
255 }
256 return Register();
257}
258
259Register fastEmit_ARMISD_CMPFPw0_r(MVT VT, MVT RetVT, Register Op0) {
260 switch (VT.SimpleTy) {
261 case MVT::f16: return fastEmit_ARMISD_CMPFPw0_MVT_f16_r(RetVT, Op0);
262 case MVT::f32: return fastEmit_ARMISD_CMPFPw0_MVT_f32_r(RetVT, Op0);
263 case MVT::f64: return fastEmit_ARMISD_CMPFPw0_MVT_f64_r(RetVT, Op0);
264 default: return Register();
265 }
266}
267
268// FastEmit functions for ARMISD::VADDVs.
269
270Register fastEmit_ARMISD_VADDVs_MVT_v16i8_r(MVT RetVT, Register Op0) {
271 if (RetVT.SimpleTy != MVT::i32)
272 return Register();
273 if ((Subtarget->hasMVEIntegerOps())) {
274 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVs8no_acc, RC: &ARM::tGPREvenRegClass, Op0);
275 }
276 return Register();
277}
278
279Register fastEmit_ARMISD_VADDVs_MVT_v8i16_r(MVT RetVT, Register Op0) {
280 if (RetVT.SimpleTy != MVT::i32)
281 return Register();
282 if ((Subtarget->hasMVEIntegerOps())) {
283 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVs16no_acc, RC: &ARM::tGPREvenRegClass, Op0);
284 }
285 return Register();
286}
287
288Register fastEmit_ARMISD_VADDVs_MVT_v4i32_r(MVT RetVT, Register Op0) {
289 if (RetVT.SimpleTy != MVT::i32)
290 return Register();
291 if ((Subtarget->hasMVEIntegerOps())) {
292 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVs32no_acc, RC: &ARM::tGPREvenRegClass, Op0);
293 }
294 return Register();
295}
296
297Register fastEmit_ARMISD_VADDVs_r(MVT VT, MVT RetVT, Register Op0) {
298 switch (VT.SimpleTy) {
299 case MVT::v16i8: return fastEmit_ARMISD_VADDVs_MVT_v16i8_r(RetVT, Op0);
300 case MVT::v8i16: return fastEmit_ARMISD_VADDVs_MVT_v8i16_r(RetVT, Op0);
301 case MVT::v4i32: return fastEmit_ARMISD_VADDVs_MVT_v4i32_r(RetVT, Op0);
302 default: return Register();
303 }
304}
305
306// FastEmit functions for ARMISD::VADDVu.
307
308Register fastEmit_ARMISD_VADDVu_MVT_v16i8_r(MVT RetVT, Register Op0) {
309 if (RetVT.SimpleTy != MVT::i32)
310 return Register();
311 if ((Subtarget->hasMVEIntegerOps())) {
312 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu8no_acc, RC: &ARM::tGPREvenRegClass, Op0);
313 }
314 return Register();
315}
316
317Register fastEmit_ARMISD_VADDVu_MVT_v8i16_r(MVT RetVT, Register Op0) {
318 if (RetVT.SimpleTy != MVT::i32)
319 return Register();
320 if ((Subtarget->hasMVEIntegerOps())) {
321 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu16no_acc, RC: &ARM::tGPREvenRegClass, Op0);
322 }
323 return Register();
324}
325
326Register fastEmit_ARMISD_VADDVu_MVT_v4i32_r(MVT RetVT, Register Op0) {
327 if (RetVT.SimpleTy != MVT::i32)
328 return Register();
329 if ((Subtarget->hasMVEIntegerOps())) {
330 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu32no_acc, RC: &ARM::tGPREvenRegClass, Op0);
331 }
332 return Register();
333}
334
335Register fastEmit_ARMISD_VADDVu_r(MVT VT, MVT RetVT, Register Op0) {
336 switch (VT.SimpleTy) {
337 case MVT::v16i8: return fastEmit_ARMISD_VADDVu_MVT_v16i8_r(RetVT, Op0);
338 case MVT::v8i16: return fastEmit_ARMISD_VADDVu_MVT_v8i16_r(RetVT, Op0);
339 case MVT::v4i32: return fastEmit_ARMISD_VADDVu_MVT_v4i32_r(RetVT, Op0);
340 default: return Register();
341 }
342}
343
344// FastEmit functions for ARMISD::VDUP.
345
346Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i8_r(Register Op0) {
347 if ((Subtarget->hasNEON())) {
348 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP8d, RC: &ARM::DPRRegClass, Op0);
349 }
350 return Register();
351}
352
353Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v16i8_r(Register Op0) {
354 if ((Subtarget->hasMVEIntegerOps())) {
355 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP8, RC: &ARM::MQPRRegClass, Op0);
356 }
357 if ((Subtarget->hasNEON())) {
358 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP8q, RC: &ARM::QPRRegClass, Op0);
359 }
360 return Register();
361}
362
363Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i16_r(Register Op0) {
364 if ((Subtarget->hasNEON())) {
365 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP16d, RC: &ARM::DPRRegClass, Op0);
366 }
367 return Register();
368}
369
370Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i16_r(Register Op0) {
371 if ((Subtarget->hasMVEIntegerOps())) {
372 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP16, RC: &ARM::MQPRRegClass, Op0);
373 }
374 if ((Subtarget->hasNEON())) {
375 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP16q, RC: &ARM::QPRRegClass, Op0);
376 }
377 return Register();
378}
379
380Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v2i32_r(Register Op0) {
381 if ((!Subtarget->hasSlowVDUP32()) && (Subtarget->hasNEON())) {
382 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP32d, RC: &ARM::DPRRegClass, Op0);
383 }
384 return Register();
385}
386
387Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i32_r(Register Op0) {
388 if ((Subtarget->hasMVEIntegerOps())) {
389 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP32, RC: &ARM::MQPRRegClass, Op0);
390 }
391 if ((Subtarget->hasNEON())) {
392 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP32q, RC: &ARM::QPRRegClass, Op0);
393 }
394 return Register();
395}
396
397Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8f16_r(Register Op0) {
398 if ((Subtarget->hasMVEIntegerOps())) {
399 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP16, RC: &ARM::MQPRRegClass, Op0);
400 }
401 return Register();
402}
403
404Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4f32_r(Register Op0) {
405 if ((Subtarget->hasMVEIntegerOps())) {
406 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP32, RC: &ARM::MQPRRegClass, Op0);
407 }
408 return Register();
409}
410
411Register fastEmit_ARMISD_VDUP_MVT_i32_r(MVT RetVT, Register Op0) {
412switch (RetVT.SimpleTy) {
413 case MVT::v8i8: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i8_r(Op0);
414 case MVT::v16i8: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v16i8_r(Op0);
415 case MVT::v4i16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i16_r(Op0);
416 case MVT::v8i16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i16_r(Op0);
417 case MVT::v2i32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v2i32_r(Op0);
418 case MVT::v4i32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i32_r(Op0);
419 case MVT::v8f16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8f16_r(Op0);
420 case MVT::v4f32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4f32_r(Op0);
421 default: return Register();
422}
423}
424
425Register fastEmit_ARMISD_VDUP_r(MVT VT, MVT RetVT, Register Op0) {
426 switch (VT.SimpleTy) {
427 case MVT::i32: return fastEmit_ARMISD_VDUP_MVT_i32_r(RetVT, Op0);
428 default: return Register();
429 }
430}
431
432// FastEmit functions for ARMISD::VMOVSR.
433
434Register fastEmit_ARMISD_VMOVSR_MVT_i32_r(MVT RetVT, Register Op0) {
435 if (RetVT.SimpleTy != MVT::f32)
436 return Register();
437 if ((Subtarget->hasFPRegs()) && (Subtarget->preferVMOVSR() ||!Subtarget->useNEONForSinglePrecisionFP())) {
438 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVSR, RC: &ARM::SPRRegClass, Op0);
439 }
440 return Register();
441}
442
443Register fastEmit_ARMISD_VMOVSR_r(MVT VT, MVT RetVT, Register Op0) {
444 switch (VT.SimpleTy) {
445 case MVT::i32: return fastEmit_ARMISD_VMOVSR_MVT_i32_r(RetVT, Op0);
446 default: return Register();
447 }
448}
449
450// FastEmit functions for ARMISD::VMOVhr.
451
452Register fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_bf16_r(Register Op0) {
453 if ((Subtarget->hasFPRegs16())) {
454 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVHR, RC: &ARM::HPRRegClass, Op0);
455 }
456 return Register();
457}
458
459Register fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_f16_r(Register Op0) {
460 if ((Subtarget->hasFPRegs16())) {
461 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVHR, RC: &ARM::HPRRegClass, Op0);
462 }
463 return Register();
464}
465
466Register fastEmit_ARMISD_VMOVhr_MVT_i32_r(MVT RetVT, Register Op0) {
467switch (RetVT.SimpleTy) {
468 case MVT::bf16: return fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_bf16_r(Op0);
469 case MVT::f16: return fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_f16_r(Op0);
470 default: return Register();
471}
472}
473
474Register fastEmit_ARMISD_VMOVhr_r(MVT VT, MVT RetVT, Register Op0) {
475 switch (VT.SimpleTy) {
476 case MVT::i32: return fastEmit_ARMISD_VMOVhr_MVT_i32_r(RetVT, Op0);
477 default: return Register();
478 }
479}
480
481// FastEmit functions for ARMISD::VMOVrh.
482
483Register fastEmit_ARMISD_VMOVrh_MVT_bf16_r(MVT RetVT, Register Op0) {
484 if (RetVT.SimpleTy != MVT::i32)
485 return Register();
486 if ((Subtarget->hasFPRegs16())) {
487 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVRH, RC: &ARM::rGPRRegClass, Op0);
488 }
489 return Register();
490}
491
492Register fastEmit_ARMISD_VMOVrh_MVT_f16_r(MVT RetVT, Register Op0) {
493 if (RetVT.SimpleTy != MVT::i32)
494 return Register();
495 if ((Subtarget->hasFPRegs16())) {
496 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVRH, RC: &ARM::rGPRRegClass, Op0);
497 }
498 return Register();
499}
500
501Register fastEmit_ARMISD_VMOVrh_r(MVT VT, MVT RetVT, Register Op0) {
502 switch (VT.SimpleTy) {
503 case MVT::bf16: return fastEmit_ARMISD_VMOVrh_MVT_bf16_r(RetVT, Op0);
504 case MVT::f16: return fastEmit_ARMISD_VMOVrh_MVT_f16_r(RetVT, Op0);
505 default: return Register();
506 }
507}
508
509// FastEmit functions for ARMISD::VREV16.
510
511Register fastEmit_ARMISD_VREV16_MVT_v8i8_r(MVT RetVT, Register Op0) {
512 if (RetVT.SimpleTy != MVT::v8i8)
513 return Register();
514 if ((Subtarget->hasNEON())) {
515 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
516 }
517 return Register();
518}
519
520Register fastEmit_ARMISD_VREV16_MVT_v16i8_r(MVT RetVT, Register Op0) {
521 if (RetVT.SimpleTy != MVT::v16i8)
522 return Register();
523 if ((Subtarget->hasMVEIntegerOps())) {
524 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
525 }
526 if ((Subtarget->hasNEON())) {
527 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
528 }
529 return Register();
530}
531
532Register fastEmit_ARMISD_VREV16_r(MVT VT, MVT RetVT, Register Op0) {
533 switch (VT.SimpleTy) {
534 case MVT::v8i8: return fastEmit_ARMISD_VREV16_MVT_v8i8_r(RetVT, Op0);
535 case MVT::v16i8: return fastEmit_ARMISD_VREV16_MVT_v16i8_r(RetVT, Op0);
536 default: return Register();
537 }
538}
539
540// FastEmit functions for ARMISD::VREV32.
541
542Register fastEmit_ARMISD_VREV32_MVT_v8i8_r(MVT RetVT, Register Op0) {
543 if (RetVT.SimpleTy != MVT::v8i8)
544 return Register();
545 if ((Subtarget->hasNEON())) {
546 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
547 }
548 return Register();
549}
550
551Register fastEmit_ARMISD_VREV32_MVT_v16i8_r(MVT RetVT, Register Op0) {
552 if (RetVT.SimpleTy != MVT::v16i8)
553 return Register();
554 if ((Subtarget->hasMVEIntegerOps())) {
555 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
556 }
557 if ((Subtarget->hasNEON())) {
558 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
559 }
560 return Register();
561}
562
563Register fastEmit_ARMISD_VREV32_MVT_v4i16_r(MVT RetVT, Register Op0) {
564 if (RetVT.SimpleTy != MVT::v4i16)
565 return Register();
566 if ((Subtarget->hasNEON())) {
567 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
568 }
569 return Register();
570}
571
572Register fastEmit_ARMISD_VREV32_MVT_v8i16_r(MVT RetVT, Register Op0) {
573 if (RetVT.SimpleTy != MVT::v8i16)
574 return Register();
575 if ((Subtarget->hasMVEIntegerOps())) {
576 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
577 }
578 if ((Subtarget->hasNEON())) {
579 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
580 }
581 return Register();
582}
583
584Register fastEmit_ARMISD_VREV32_MVT_v4f16_r(MVT RetVT, Register Op0) {
585 if (RetVT.SimpleTy != MVT::v4f16)
586 return Register();
587 if ((Subtarget->hasNEON())) {
588 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
589 }
590 return Register();
591}
592
593Register fastEmit_ARMISD_VREV32_MVT_v8f16_r(MVT RetVT, Register Op0) {
594 if (RetVT.SimpleTy != MVT::v8f16)
595 return Register();
596 if ((Subtarget->hasMVEIntegerOps())) {
597 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
598 }
599 if ((Subtarget->hasNEON())) {
600 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
601 }
602 return Register();
603}
604
605Register fastEmit_ARMISD_VREV32_MVT_v4bf16_r(MVT RetVT, Register Op0) {
606 if (RetVT.SimpleTy != MVT::v4bf16)
607 return Register();
608 if ((Subtarget->hasNEON())) {
609 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
610 }
611 return Register();
612}
613
614Register fastEmit_ARMISD_VREV32_MVT_v8bf16_r(MVT RetVT, Register Op0) {
615 if (RetVT.SimpleTy != MVT::v8bf16)
616 return Register();
617 if ((Subtarget->hasNEON())) {
618 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
619 }
620 return Register();
621}
622
623Register fastEmit_ARMISD_VREV32_r(MVT VT, MVT RetVT, Register Op0) {
624 switch (VT.SimpleTy) {
625 case MVT::v8i8: return fastEmit_ARMISD_VREV32_MVT_v8i8_r(RetVT, Op0);
626 case MVT::v16i8: return fastEmit_ARMISD_VREV32_MVT_v16i8_r(RetVT, Op0);
627 case MVT::v4i16: return fastEmit_ARMISD_VREV32_MVT_v4i16_r(RetVT, Op0);
628 case MVT::v8i16: return fastEmit_ARMISD_VREV32_MVT_v8i16_r(RetVT, Op0);
629 case MVT::v4f16: return fastEmit_ARMISD_VREV32_MVT_v4f16_r(RetVT, Op0);
630 case MVT::v8f16: return fastEmit_ARMISD_VREV32_MVT_v8f16_r(RetVT, Op0);
631 case MVT::v4bf16: return fastEmit_ARMISD_VREV32_MVT_v4bf16_r(RetVT, Op0);
632 case MVT::v8bf16: return fastEmit_ARMISD_VREV32_MVT_v8bf16_r(RetVT, Op0);
633 default: return Register();
634 }
635}
636
637// FastEmit functions for ARMISD::VREV64.
638
639Register fastEmit_ARMISD_VREV64_MVT_v8i8_r(MVT RetVT, Register Op0) {
640 if (RetVT.SimpleTy != MVT::v8i8)
641 return Register();
642 if ((Subtarget->hasNEON())) {
643 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
644 }
645 return Register();
646}
647
648Register fastEmit_ARMISD_VREV64_MVT_v16i8_r(MVT RetVT, Register Op0) {
649 if (RetVT.SimpleTy != MVT::v16i8)
650 return Register();
651 if ((Subtarget->hasMVEIntegerOps())) {
652 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
653 }
654 if ((Subtarget->hasNEON())) {
655 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
656 }
657 return Register();
658}
659
660Register fastEmit_ARMISD_VREV64_MVT_v4i16_r(MVT RetVT, Register Op0) {
661 if (RetVT.SimpleTy != MVT::v4i16)
662 return Register();
663 if ((Subtarget->hasNEON())) {
664 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
665 }
666 return Register();
667}
668
669Register fastEmit_ARMISD_VREV64_MVT_v8i16_r(MVT RetVT, Register Op0) {
670 if (RetVT.SimpleTy != MVT::v8i16)
671 return Register();
672 if ((Subtarget->hasMVEIntegerOps())) {
673 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
674 }
675 if ((Subtarget->hasNEON())) {
676 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
677 }
678 return Register();
679}
680
681Register fastEmit_ARMISD_VREV64_MVT_v2i32_r(MVT RetVT, Register Op0) {
682 if (RetVT.SimpleTy != MVT::v2i32)
683 return Register();
684 if ((Subtarget->hasNEON())) {
685 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
686 }
687 return Register();
688}
689
690Register fastEmit_ARMISD_VREV64_MVT_v4i32_r(MVT RetVT, Register Op0) {
691 if (RetVT.SimpleTy != MVT::v4i32)
692 return Register();
693 if ((Subtarget->hasMVEIntegerOps())) {
694 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
695 }
696 if ((Subtarget->hasNEON())) {
697 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
698 }
699 return Register();
700}
701
702Register fastEmit_ARMISD_VREV64_MVT_v4f16_r(MVT RetVT, Register Op0) {
703 if (RetVT.SimpleTy != MVT::v4f16)
704 return Register();
705 if ((Subtarget->hasNEON())) {
706 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
707 }
708 return Register();
709}
710
711Register fastEmit_ARMISD_VREV64_MVT_v8f16_r(MVT RetVT, Register Op0) {
712 if (RetVT.SimpleTy != MVT::v8f16)
713 return Register();
714 if ((Subtarget->hasMVEIntegerOps())) {
715 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
716 }
717 if ((Subtarget->hasNEON())) {
718 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
719 }
720 return Register();
721}
722
723Register fastEmit_ARMISD_VREV64_MVT_v4bf16_r(MVT RetVT, Register Op0) {
724 if (RetVT.SimpleTy != MVT::v4bf16)
725 return Register();
726 if ((Subtarget->hasNEON())) {
727 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
728 }
729 return Register();
730}
731
732Register fastEmit_ARMISD_VREV64_MVT_v8bf16_r(MVT RetVT, Register Op0) {
733 if (RetVT.SimpleTy != MVT::v8bf16)
734 return Register();
735 if ((Subtarget->hasNEON())) {
736 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
737 }
738 return Register();
739}
740
741Register fastEmit_ARMISD_VREV64_MVT_v2f32_r(MVT RetVT, Register Op0) {
742 if (RetVT.SimpleTy != MVT::v2f32)
743 return Register();
744 if ((Subtarget->hasNEON())) {
745 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
746 }
747 return Register();
748}
749
750Register fastEmit_ARMISD_VREV64_MVT_v4f32_r(MVT RetVT, Register Op0) {
751 if (RetVT.SimpleTy != MVT::v4f32)
752 return Register();
753 if ((Subtarget->hasMVEIntegerOps())) {
754 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
755 }
756 if ((Subtarget->hasNEON())) {
757 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
758 }
759 return Register();
760}
761
762Register fastEmit_ARMISD_VREV64_r(MVT VT, MVT RetVT, Register Op0) {
763 switch (VT.SimpleTy) {
764 case MVT::v8i8: return fastEmit_ARMISD_VREV64_MVT_v8i8_r(RetVT, Op0);
765 case MVT::v16i8: return fastEmit_ARMISD_VREV64_MVT_v16i8_r(RetVT, Op0);
766 case MVT::v4i16: return fastEmit_ARMISD_VREV64_MVT_v4i16_r(RetVT, Op0);
767 case MVT::v8i16: return fastEmit_ARMISD_VREV64_MVT_v8i16_r(RetVT, Op0);
768 case MVT::v2i32: return fastEmit_ARMISD_VREV64_MVT_v2i32_r(RetVT, Op0);
769 case MVT::v4i32: return fastEmit_ARMISD_VREV64_MVT_v4i32_r(RetVT, Op0);
770 case MVT::v4f16: return fastEmit_ARMISD_VREV64_MVT_v4f16_r(RetVT, Op0);
771 case MVT::v8f16: return fastEmit_ARMISD_VREV64_MVT_v8f16_r(RetVT, Op0);
772 case MVT::v4bf16: return fastEmit_ARMISD_VREV64_MVT_v4bf16_r(RetVT, Op0);
773 case MVT::v8bf16: return fastEmit_ARMISD_VREV64_MVT_v8bf16_r(RetVT, Op0);
774 case MVT::v2f32: return fastEmit_ARMISD_VREV64_MVT_v2f32_r(RetVT, Op0);
775 case MVT::v4f32: return fastEmit_ARMISD_VREV64_MVT_v4f32_r(RetVT, Op0);
776 default: return Register();
777 }
778}
779
780// FastEmit functions for ARMISD::WIN__DBZCHK.
781
782Register fastEmit_ARMISD_WIN__DBZCHK_MVT_i32_r(MVT RetVT, Register Op0) {
783 if (RetVT.SimpleTy != MVT::isVoid)
784 return Register();
785 return fastEmitInst_r(MachineInstOpcode: ARM::WIN__DBZCHK, RC: &ARM::tGPRRegClass, Op0);
786}
787
788Register fastEmit_ARMISD_WIN__DBZCHK_r(MVT VT, MVT RetVT, Register Op0) {
789 switch (VT.SimpleTy) {
790 case MVT::i32: return fastEmit_ARMISD_WIN__DBZCHK_MVT_i32_r(RetVT, Op0);
791 default: return Register();
792 }
793}
794
795// FastEmit functions for ARMISD::WLSSETUP.
796
797Register fastEmit_ARMISD_WLSSETUP_MVT_i32_r(MVT RetVT, Register Op0) {
798 if (RetVT.SimpleTy != MVT::i32)
799 return Register();
800 if ((Subtarget->hasLOB()) && (Subtarget->hasV8_1MMainlineOps()) && (Subtarget->isThumb2())) {
801 return fastEmitInst_r(MachineInstOpcode: ARM::t2WhileLoopSetup, RC: &ARM::GPRlrRegClass, Op0);
802 }
803 return Register();
804}
805
806Register fastEmit_ARMISD_WLSSETUP_r(MVT VT, MVT RetVT, Register Op0) {
807 switch (VT.SimpleTy) {
808 case MVT::i32: return fastEmit_ARMISD_WLSSETUP_MVT_i32_r(RetVT, Op0);
809 default: return Register();
810 }
811}
812
813// FastEmit functions for ARMISD::tSECALL.
814
815Register fastEmit_ARMISD_tSECALL_MVT_i32_r(MVT RetVT, Register Op0) {
816 if (RetVT.SimpleTy != MVT::isVoid)
817 return Register();
818 if ((Subtarget->has8MSecExt()) && (Subtarget->isThumb())) {
819 return fastEmitInst_r(MachineInstOpcode: ARM::tBLXNS_CALL, RC: &ARM::GPRnopcRegClass, Op0);
820 }
821 return Register();
822}
823
824Register fastEmit_ARMISD_tSECALL_r(MVT VT, MVT RetVT, Register Op0) {
825 switch (VT.SimpleTy) {
826 case MVT::i32: return fastEmit_ARMISD_tSECALL_MVT_i32_r(RetVT, Op0);
827 default: return Register();
828 }
829}
830
831// FastEmit functions for ISD::ABS.
832
833Register fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, Register Op0) {
834 if (RetVT.SimpleTy != MVT::v8i8)
835 return Register();
836 if ((Subtarget->hasNEON())) {
837 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv8i8, RC: &ARM::DPRRegClass, Op0);
838 }
839 return Register();
840}
841
842Register fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, Register Op0) {
843 if (RetVT.SimpleTy != MVT::v16i8)
844 return Register();
845 if ((Subtarget->hasMVEIntegerOps())) {
846 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSs8, RC: &ARM::MQPRRegClass, Op0);
847 }
848 if ((Subtarget->hasNEON())) {
849 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv16i8, RC: &ARM::QPRRegClass, Op0);
850 }
851 return Register();
852}
853
854Register fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, Register Op0) {
855 if (RetVT.SimpleTy != MVT::v4i16)
856 return Register();
857 if ((Subtarget->hasNEON())) {
858 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv4i16, RC: &ARM::DPRRegClass, Op0);
859 }
860 return Register();
861}
862
863Register fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, Register Op0) {
864 if (RetVT.SimpleTy != MVT::v8i16)
865 return Register();
866 if ((Subtarget->hasMVEIntegerOps())) {
867 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSs16, RC: &ARM::MQPRRegClass, Op0);
868 }
869 if ((Subtarget->hasNEON())) {
870 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv8i16, RC: &ARM::QPRRegClass, Op0);
871 }
872 return Register();
873}
874
875Register fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, Register Op0) {
876 if (RetVT.SimpleTy != MVT::v2i32)
877 return Register();
878 if ((Subtarget->hasNEON())) {
879 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv2i32, RC: &ARM::DPRRegClass, Op0);
880 }
881 return Register();
882}
883
884Register fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, Register Op0) {
885 if (RetVT.SimpleTy != MVT::v4i32)
886 return Register();
887 if ((Subtarget->hasMVEIntegerOps())) {
888 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSs32, RC: &ARM::MQPRRegClass, Op0);
889 }
890 if ((Subtarget->hasNEON())) {
891 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv4i32, RC: &ARM::QPRRegClass, Op0);
892 }
893 return Register();
894}
895
896Register fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, Register Op0) {
897 switch (VT.SimpleTy) {
898 case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0);
899 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
900 case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0);
901 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
902 case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0);
903 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
904 default: return Register();
905 }
906}
907
908// FastEmit functions for ISD::ANY_EXTEND.
909
910Register fastEmit_ISD_ANY_EXTEND_MVT_v8i8_r(MVT RetVT, Register Op0) {
911 if (RetVT.SimpleTy != MVT::v8i16)
912 return Register();
913 if ((Subtarget->hasNEON())) {
914 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv8i16, RC: &ARM::QPRRegClass, Op0);
915 }
916 return Register();
917}
918
919Register fastEmit_ISD_ANY_EXTEND_MVT_v4i16_r(MVT RetVT, Register Op0) {
920 if (RetVT.SimpleTy != MVT::v4i32)
921 return Register();
922 if ((Subtarget->hasNEON())) {
923 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv4i32, RC: &ARM::QPRRegClass, Op0);
924 }
925 return Register();
926}
927
928Register fastEmit_ISD_ANY_EXTEND_MVT_v2i32_r(MVT RetVT, Register Op0) {
929 if (RetVT.SimpleTy != MVT::v2i64)
930 return Register();
931 if ((Subtarget->hasNEON())) {
932 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv2i64, RC: &ARM::QPRRegClass, Op0);
933 }
934 return Register();
935}
936
937Register fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
938 switch (VT.SimpleTy) {
939 case MVT::v8i8: return fastEmit_ISD_ANY_EXTEND_MVT_v8i8_r(RetVT, Op0);
940 case MVT::v4i16: return fastEmit_ISD_ANY_EXTEND_MVT_v4i16_r(RetVT, Op0);
941 case MVT::v2i32: return fastEmit_ISD_ANY_EXTEND_MVT_v2i32_r(RetVT, Op0);
942 default: return Register();
943 }
944}
945
946// FastEmit functions for ISD::BITCAST.
947
948Register fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, Register Op0) {
949 if (RetVT.SimpleTy != MVT::f32)
950 return Register();
951 if ((Subtarget->hasFPRegs()) && (Subtarget->preferVMOVSR() ||!Subtarget->useNEONForSinglePrecisionFP())) {
952 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVSR, RC: &ARM::SPRRegClass, Op0);
953 }
954 return Register();
955}
956
957Register fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, Register Op0) {
958 if (RetVT.SimpleTy != MVT::i32)
959 return Register();
960 if ((Subtarget->hasFPRegs())) {
961 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVRS, RC: &ARM::GPRRegClass, Op0);
962 }
963 return Register();
964}
965
966Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Register Op0) {
967 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
968 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
969 }
970 return Register();
971}
972
973Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Register Op0) {
974 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
975 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
976 }
977 return Register();
978}
979
980Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Register Op0) {
981 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
982 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
983 }
984 return Register();
985}
986
987Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Register Op0) {
988 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
989 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
990 }
991 return Register();
992}
993
994Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Register Op0) {
995 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
996 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
997 }
998 return Register();
999}
1000
1001Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Register Op0) {
1002 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1003 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1004 }
1005 return Register();
1006}
1007
1008Register fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, Register Op0) {
1009switch (RetVT.SimpleTy) {
1010 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0);
1011 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0);
1012 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0);
1013 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0);
1014 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Op0);
1015 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0);
1016 default: return Register();
1017}
1018}
1019
1020Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Register Op0) {
1021 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1022 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
1023 }
1024 return Register();
1025}
1026
1027Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Register Op0) {
1028 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1029 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1030 }
1031 return Register();
1032}
1033
1034Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Register Op0) {
1035 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1036 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1037 }
1038 return Register();
1039}
1040
1041Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Register Op0) {
1042 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1043 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
1044 }
1045 return Register();
1046}
1047
1048Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Register Op0) {
1049 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1050 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1051 }
1052 return Register();
1053}
1054
1055Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Register Op0) {
1056 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1057 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1058 }
1059 return Register();
1060}
1061
1062Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Register Op0) {
1063 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1064 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1065 }
1066 return Register();
1067}
1068
1069Register fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, Register Op0) {
1070switch (RetVT.SimpleTy) {
1071 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0);
1072 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0);
1073 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0);
1074 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0);
1075 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0);
1076 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Op0);
1077 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0);
1078 default: return Register();
1079}
1080}
1081
1082Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Register Op0) {
1083 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1084 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1085 }
1086 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1087 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1088 }
1089 return Register();
1090}
1091
1092Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Register Op0) {
1093 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1094 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1095 }
1096 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1097 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1098 }
1099 return Register();
1100}
1101
1102Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Register Op0) {
1103 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1104 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1105 }
1106 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1107 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1108 }
1109 return Register();
1110}
1111
1112Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Register Op0) {
1113 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1114 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1115 }
1116 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1117 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1118 }
1119 return Register();
1120}
1121
1122Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Register Op0) {
1123 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1124 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1125 }
1126 return Register();
1127}
1128
1129Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Register Op0) {
1130 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1131 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1132 }
1133 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1134 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1135 }
1136 return Register();
1137}
1138
1139Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Register Op0) {
1140 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1141 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1142 }
1143 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1144 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1145 }
1146 return Register();
1147}
1148
1149Register fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, Register Op0) {
1150switch (RetVT.SimpleTy) {
1151 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0);
1152 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0);
1153 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0);
1154 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0);
1155 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Op0);
1156 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0);
1157 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0);
1158 default: return Register();
1159}
1160}
1161
1162Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Register Op0) {
1163 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1164 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1165 }
1166 return Register();
1167}
1168
1169Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Register Op0) {
1170 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1171 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1172 }
1173 return Register();
1174}
1175
1176Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Register Op0) {
1177 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1178 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1179 }
1180 return Register();
1181}
1182
1183Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Register Op0) {
1184 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1185 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1186 }
1187 return Register();
1188}
1189
1190Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Register Op0) {
1191 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1192 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1193 }
1194 return Register();
1195}
1196
1197Register fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, Register Op0) {
1198switch (RetVT.SimpleTy) {
1199 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0);
1200 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0);
1201 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0);
1202 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0);
1203 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0);
1204 default: return Register();
1205}
1206}
1207
1208Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Register Op0) {
1209 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1210 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1211 }
1212 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1213 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1214 }
1215 return Register();
1216}
1217
1218Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Register Op0) {
1219 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1220 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1221 }
1222 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1223 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1224 }
1225 return Register();
1226}
1227
1228Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Register Op0) {
1229 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1230 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1231 }
1232 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1233 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1234 }
1235 return Register();
1236}
1237
1238Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Register Op0) {
1239 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1240 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1241 }
1242 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1243 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1244 }
1245 return Register();
1246}
1247
1248Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Register Op0) {
1249 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1250 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1251 }
1252 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1253 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1254 }
1255 return Register();
1256}
1257
1258Register fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, Register Op0) {
1259switch (RetVT.SimpleTy) {
1260 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0);
1261 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0);
1262 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0);
1263 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0);
1264 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0);
1265 default: return Register();
1266}
1267}
1268
1269Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Register Op0) {
1270 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1271 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1272 }
1273 return Register();
1274}
1275
1276Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Register Op0) {
1277 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1278 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1279 }
1280 return Register();
1281}
1282
1283Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Register Op0) {
1284 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1285 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1286 }
1287 return Register();
1288}
1289
1290Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Register Op0) {
1291 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1292 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1293 }
1294 return Register();
1295}
1296
1297Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Register Op0) {
1298 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1299 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1300 }
1301 return Register();
1302}
1303
1304Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Register Op0) {
1305 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1306 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1307 }
1308 return Register();
1309}
1310
1311Register fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, Register Op0) {
1312switch (RetVT.SimpleTy) {
1313 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0);
1314 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0);
1315 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0);
1316 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0);
1317 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0);
1318 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Op0);
1319 default: return Register();
1320}
1321}
1322
1323Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Register Op0) {
1324 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1325 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1326 }
1327 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1328 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1329 }
1330 return Register();
1331}
1332
1333Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Register Op0) {
1334 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1335 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1336 }
1337 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1338 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1339 }
1340 return Register();
1341}
1342
1343Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Register Op0) {
1344 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1345 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1346 }
1347 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1348 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1349 }
1350 return Register();
1351}
1352
1353Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Register Op0) {
1354 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1355 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1356 }
1357 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1358 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1359 }
1360 return Register();
1361}
1362
1363Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Register Op0) {
1364 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1365 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1366 }
1367 return Register();
1368}
1369
1370Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Register Op0) {
1371 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1372 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1373 }
1374 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1375 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1376 }
1377 return Register();
1378}
1379
1380Register fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, Register Op0) {
1381switch (RetVT.SimpleTy) {
1382 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0);
1383 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0);
1384 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0);
1385 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0);
1386 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Op0);
1387 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0);
1388 default: return Register();
1389}
1390}
1391
1392Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Register Op0) {
1393 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1394 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
1395 }
1396 return Register();
1397}
1398
1399Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Register Op0) {
1400 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1401 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1402 }
1403 return Register();
1404}
1405
1406Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Register Op0) {
1407 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1408 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1409 }
1410 return Register();
1411}
1412
1413Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Register Op0) {
1414 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1415 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1416 }
1417 return Register();
1418}
1419
1420Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Register Op0) {
1421 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1422 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1423 }
1424 return Register();
1425}
1426
1427Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Register Op0) {
1428 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1429 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1430 }
1431 return Register();
1432}
1433
1434Register fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, Register Op0) {
1435switch (RetVT.SimpleTy) {
1436 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0);
1437 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0);
1438 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0);
1439 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0);
1440 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Op0);
1441 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0);
1442 default: return Register();
1443}
1444}
1445
1446Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Register Op0) {
1447 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1448 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1449 }
1450 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1451 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1452 }
1453 return Register();
1454}
1455
1456Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Register Op0) {
1457 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1458 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1459 }
1460 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1461 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1462 }
1463 return Register();
1464}
1465
1466Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Register Op0) {
1467 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1468 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1469 }
1470 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1471 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1472 }
1473 return Register();
1474}
1475
1476Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Register Op0) {
1477 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1478 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1479 }
1480 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1481 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1482 }
1483 return Register();
1484}
1485
1486Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Register Op0) {
1487 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1488 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1489 }
1490 return Register();
1491}
1492
1493Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Register Op0) {
1494 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1495 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1496 }
1497 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1498 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1499 }
1500 return Register();
1501}
1502
1503Register fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, Register Op0) {
1504switch (RetVT.SimpleTy) {
1505 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0);
1506 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0);
1507 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0);
1508 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0);
1509 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Op0);
1510 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0);
1511 default: return Register();
1512}
1513}
1514
1515Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Register Op0) {
1516 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1517 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1518 }
1519 return Register();
1520}
1521
1522Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Register Op0) {
1523 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1524 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1525 }
1526 return Register();
1527}
1528
1529Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Register Op0) {
1530 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1531 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1532 }
1533 return Register();
1534}
1535
1536Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Register Op0) {
1537 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1538 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1539 }
1540 return Register();
1541}
1542
1543Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Register Op0) {
1544 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1545 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1546 }
1547 return Register();
1548}
1549
1550Register fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, Register Op0) {
1551switch (RetVT.SimpleTy) {
1552 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0);
1553 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0);
1554 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0);
1555 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0);
1556 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0);
1557 default: return Register();
1558}
1559}
1560
1561Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Register Op0) {
1562 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1563 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1564 }
1565 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1566 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1567 }
1568 return Register();
1569}
1570
1571Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Register Op0) {
1572 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1573 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1574 }
1575 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1576 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1577 }
1578 return Register();
1579}
1580
1581Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Register Op0) {
1582 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1583 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1584 }
1585 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1586 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1587 }
1588 return Register();
1589}
1590
1591Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Register Op0) {
1592 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1593 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1594 }
1595 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1596 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1597 }
1598 return Register();
1599}
1600
1601Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Register Op0) {
1602 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1603 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1604 }
1605 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1606 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1607 }
1608 return Register();
1609}
1610
1611Register fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, Register Op0) {
1612switch (RetVT.SimpleTy) {
1613 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0);
1614 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0);
1615 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0);
1616 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0);
1617 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0);
1618 default: return Register();
1619}
1620}
1621
1622Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Register Op0) {
1623 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1624 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1625 }
1626 return Register();
1627}
1628
1629Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Register Op0) {
1630 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1631 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1632 }
1633 return Register();
1634}
1635
1636Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Register Op0) {
1637 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1638 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1639 }
1640 return Register();
1641}
1642
1643Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Register Op0) {
1644 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1645 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1646 }
1647 return Register();
1648}
1649
1650Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Register Op0) {
1651 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1652 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1653 }
1654 return Register();
1655}
1656
1657Register fastEmit_ISD_BITCAST_MVT_v4bf16_r(MVT RetVT, Register Op0) {
1658switch (RetVT.SimpleTy) {
1659 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Op0);
1660 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Op0);
1661 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Op0);
1662 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Op0);
1663 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Op0);
1664 default: return Register();
1665}
1666}
1667
1668Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Register Op0) {
1669 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1670 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1671 }
1672 return Register();
1673}
1674
1675Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Register Op0) {
1676 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1677 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1678 }
1679 return Register();
1680}
1681
1682Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Register Op0) {
1683 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1684 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1685 }
1686 return Register();
1687}
1688
1689Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Register Op0) {
1690 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1691 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1692 }
1693 return Register();
1694}
1695
1696Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Register Op0) {
1697 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1698 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1699 }
1700 return Register();
1701}
1702
1703Register fastEmit_ISD_BITCAST_MVT_v8bf16_r(MVT RetVT, Register Op0) {
1704switch (RetVT.SimpleTy) {
1705 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Op0);
1706 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Op0);
1707 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Op0);
1708 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Op0);
1709 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Op0);
1710 default: return Register();
1711}
1712}
1713
1714Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Register Op0) {
1715 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1716 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1717 }
1718 return Register();
1719}
1720
1721Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Register Op0) {
1722 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1723 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1724 }
1725 return Register();
1726}
1727
1728Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Register Op0) {
1729 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1730 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1731 }
1732 return Register();
1733}
1734
1735Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Register Op0) {
1736 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1737 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1738 }
1739 return Register();
1740}
1741
1742Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Register Op0) {
1743 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1744 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1745 }
1746 return Register();
1747}
1748
1749Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Register Op0) {
1750 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1751 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1752 }
1753 return Register();
1754}
1755
1756Register fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, Register Op0) {
1757switch (RetVT.SimpleTy) {
1758 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0);
1759 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0);
1760 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0);
1761 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0);
1762 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0);
1763 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Op0);
1764 default: return Register();
1765}
1766}
1767
1768Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Register Op0) {
1769 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1770 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1771 }
1772 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1773 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1774 }
1775 return Register();
1776}
1777
1778Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Register Op0) {
1779 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1780 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1781 }
1782 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1783 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1784 }
1785 return Register();
1786}
1787
1788Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Register Op0) {
1789 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1790 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1791 }
1792 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1793 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1794 }
1795 return Register();
1796}
1797
1798Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Register Op0) {
1799 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1800 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1801 }
1802 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1803 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1804 }
1805 return Register();
1806}
1807
1808Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Register Op0) {
1809 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1810 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1811 }
1812 return Register();
1813}
1814
1815Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Register Op0) {
1816 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1817 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1818 }
1819 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1820 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1821 }
1822 return Register();
1823}
1824
1825Register fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, Register Op0) {
1826switch (RetVT.SimpleTy) {
1827 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0);
1828 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0);
1829 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0);
1830 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0);
1831 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Op0);
1832 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0);
1833 default: return Register();
1834}
1835}
1836
1837Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Register Op0) {
1838 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1839 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1840 }
1841 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1842 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1843 }
1844 return Register();
1845}
1846
1847Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Register Op0) {
1848 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1849 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1850 }
1851 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1852 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1853 }
1854 return Register();
1855}
1856
1857Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Register Op0) {
1858 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1859 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1860 }
1861 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1862 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1863 }
1864 return Register();
1865}
1866
1867Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Register Op0) {
1868 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1869 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1870 }
1871 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1872 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1873 }
1874 return Register();
1875}
1876
1877Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Register Op0) {
1878 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1879 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1880 }
1881 return Register();
1882}
1883
1884Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Register Op0) {
1885 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1886 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1887 }
1888 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1889 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1890 }
1891 return Register();
1892}
1893
1894Register fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, Register Op0) {
1895switch (RetVT.SimpleTy) {
1896 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0);
1897 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0);
1898 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0);
1899 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0);
1900 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Op0);
1901 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0);
1902 default: return Register();
1903}
1904}
1905
1906Register fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, Register Op0) {
1907 switch (VT.SimpleTy) {
1908 case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0);
1909 case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0);
1910 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
1911 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0);
1912 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0);
1913 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0);
1914 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0);
1915 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0);
1916 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0);
1917 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0);
1918 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0);
1919 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0);
1920 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0);
1921 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v4bf16_r(RetVT, Op0);
1922 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v8bf16_r(RetVT, Op0);
1923 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0);
1924 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0);
1925 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0);
1926 default: return Register();
1927 }
1928}
1929
1930// FastEmit functions for ISD::BITREVERSE.
1931
1932Register fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, Register Op0) {
1933 if (RetVT.SimpleTy != MVT::i32)
1934 return Register();
1935 if ((Subtarget->isThumb2())) {
1936 return fastEmitInst_r(MachineInstOpcode: ARM::t2RBIT, RC: &ARM::rGPRRegClass, Op0);
1937 }
1938 if ((Subtarget->hasV6T2Ops()) && (!Subtarget->isThumb())) {
1939 return fastEmitInst_r(MachineInstOpcode: ARM::RBIT, RC: &ARM::GPRRegClass, Op0);
1940 }
1941 return Register();
1942}
1943
1944Register fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, Register Op0) {
1945 switch (VT.SimpleTy) {
1946 case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0);
1947 default: return Register();
1948 }
1949}
1950
1951// FastEmit functions for ISD::BRIND.
1952
1953Register fastEmit_ISD_BRIND_MVT_i32_r(MVT RetVT, Register Op0) {
1954 if (RetVT.SimpleTy != MVT::isVoid)
1955 return Register();
1956 if ((Subtarget->isThumb())) {
1957 return fastEmitInst_r(MachineInstOpcode: ARM::tBRIND, RC: &ARM::GPRRegClass, Op0);
1958 }
1959 if ((!Subtarget->isThumb()) && (!Subtarget->hasV4TOps())) {
1960 return fastEmitInst_r(MachineInstOpcode: ARM::MOVPCRX, RC: &ARM::GPRRegClass, Op0);
1961 }
1962 if ((Subtarget->hasV4TOps()) && (!Subtarget->isThumb())) {
1963 return fastEmitInst_r(MachineInstOpcode: ARM::BX, RC: &ARM::GPRRegClass, Op0);
1964 }
1965 return Register();
1966}
1967
1968Register fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, Register Op0) {
1969 switch (VT.SimpleTy) {
1970 case MVT::i32: return fastEmit_ISD_BRIND_MVT_i32_r(RetVT, Op0);
1971 default: return Register();
1972 }
1973}
1974
1975// FastEmit functions for ISD::BSWAP.
1976
1977Register fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, Register Op0) {
1978 if (RetVT.SimpleTy != MVT::i32)
1979 return Register();
1980 if ((Subtarget->isThumb2())) {
1981 return fastEmitInst_r(MachineInstOpcode: ARM::t2REV, RC: &ARM::rGPRRegClass, Op0);
1982 }
1983 if ((Subtarget->hasV6Ops()) && (Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
1984 return fastEmitInst_r(MachineInstOpcode: ARM::tREV, RC: &ARM::tGPRRegClass, Op0);
1985 }
1986 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
1987 return fastEmitInst_r(MachineInstOpcode: ARM::REV, RC: &ARM::GPRRegClass, Op0);
1988 }
1989 return Register();
1990}
1991
1992Register fastEmit_ISD_BSWAP_MVT_v8i16_r(MVT RetVT, Register Op0) {
1993 if (RetVT.SimpleTy != MVT::v8i16)
1994 return Register();
1995 if ((Subtarget->hasMVEIntegerOps())) {
1996 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1997 }
1998 return Register();
1999}
2000
2001Register fastEmit_ISD_BSWAP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2002 if (RetVT.SimpleTy != MVT::v4i32)
2003 return Register();
2004 if ((Subtarget->hasMVEIntegerOps())) {
2005 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
2006 }
2007 return Register();
2008}
2009
2010Register fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, Register Op0) {
2011 switch (VT.SimpleTy) {
2012 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
2013 case MVT::v8i16: return fastEmit_ISD_BSWAP_MVT_v8i16_r(RetVT, Op0);
2014 case MVT::v4i32: return fastEmit_ISD_BSWAP_MVT_v4i32_r(RetVT, Op0);
2015 default: return Register();
2016 }
2017}
2018
2019// FastEmit functions for ISD::CTLZ.
2020
2021Register fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, Register Op0) {
2022 if (RetVT.SimpleTy != MVT::i32)
2023 return Register();
2024 if ((Subtarget->isThumb2())) {
2025 return fastEmitInst_r(MachineInstOpcode: ARM::t2CLZ, RC: &ARM::rGPRRegClass, Op0);
2026 }
2027 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb())) {
2028 return fastEmitInst_r(MachineInstOpcode: ARM::CLZ, RC: &ARM::GPRRegClass, Op0);
2029 }
2030 return Register();
2031}
2032
2033Register fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, Register Op0) {
2034 if (RetVT.SimpleTy != MVT::v8i8)
2035 return Register();
2036 if ((Subtarget->hasNEON())) {
2037 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv8i8, RC: &ARM::DPRRegClass, Op0);
2038 }
2039 return Register();
2040}
2041
2042Register fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, Register Op0) {
2043 if (RetVT.SimpleTy != MVT::v16i8)
2044 return Register();
2045 if ((Subtarget->hasMVEIntegerOps())) {
2046 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLZs8, RC: &ARM::MQPRRegClass, Op0);
2047 }
2048 if ((Subtarget->hasNEON())) {
2049 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv16i8, RC: &ARM::QPRRegClass, Op0);
2050 }
2051 return Register();
2052}
2053
2054Register fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, Register Op0) {
2055 if (RetVT.SimpleTy != MVT::v4i16)
2056 return Register();
2057 if ((Subtarget->hasNEON())) {
2058 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv4i16, RC: &ARM::DPRRegClass, Op0);
2059 }
2060 return Register();
2061}
2062
2063Register fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, Register Op0) {
2064 if (RetVT.SimpleTy != MVT::v8i16)
2065 return Register();
2066 if ((Subtarget->hasMVEIntegerOps())) {
2067 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLZs16, RC: &ARM::MQPRRegClass, Op0);
2068 }
2069 if ((Subtarget->hasNEON())) {
2070 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv8i16, RC: &ARM::QPRRegClass, Op0);
2071 }
2072 return Register();
2073}
2074
2075Register fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, Register Op0) {
2076 if (RetVT.SimpleTy != MVT::v2i32)
2077 return Register();
2078 if ((Subtarget->hasNEON())) {
2079 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv2i32, RC: &ARM::DPRRegClass, Op0);
2080 }
2081 return Register();
2082}
2083
2084Register fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, Register Op0) {
2085 if (RetVT.SimpleTy != MVT::v4i32)
2086 return Register();
2087 if ((Subtarget->hasMVEIntegerOps())) {
2088 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLZs32, RC: &ARM::MQPRRegClass, Op0);
2089 }
2090 if ((Subtarget->hasNEON())) {
2091 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv4i32, RC: &ARM::QPRRegClass, Op0);
2092 }
2093 return Register();
2094}
2095
2096Register fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, Register Op0) {
2097 switch (VT.SimpleTy) {
2098 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
2099 case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0);
2100 case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0);
2101 case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0);
2102 case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0);
2103 case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0);
2104 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
2105 default: return Register();
2106 }
2107}
2108
2109// FastEmit functions for ISD::CTPOP.
2110
2111Register fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, Register Op0) {
2112 if (RetVT.SimpleTy != MVT::v8i8)
2113 return Register();
2114 if ((Subtarget->hasNEON())) {
2115 return fastEmitInst_r(MachineInstOpcode: ARM::VCNTd, RC: &ARM::DPRRegClass, Op0);
2116 }
2117 return Register();
2118}
2119
2120Register fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, Register Op0) {
2121 if (RetVT.SimpleTy != MVT::v16i8)
2122 return Register();
2123 if ((Subtarget->hasNEON())) {
2124 return fastEmitInst_r(MachineInstOpcode: ARM::VCNTq, RC: &ARM::QPRRegClass, Op0);
2125 }
2126 return Register();
2127}
2128
2129Register fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, Register Op0) {
2130 switch (VT.SimpleTy) {
2131 case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0);
2132 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
2133 default: return Register();
2134 }
2135}
2136
2137// FastEmit functions for ISD::FABS.
2138
2139Register fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, Register Op0) {
2140 if (RetVT.SimpleTy != MVT::f16)
2141 return Register();
2142 if ((Subtarget->hasFullFP16())) {
2143 return fastEmitInst_r(MachineInstOpcode: ARM::VABSH, RC: &ARM::HPRRegClass, Op0);
2144 }
2145 return Register();
2146}
2147
2148Register fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, Register Op0) {
2149 if (RetVT.SimpleTy != MVT::f32)
2150 return Register();
2151 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
2152 return fastEmitInst_r(MachineInstOpcode: ARM::VABSS, RC: &ARM::SPRRegClass, Op0);
2153 }
2154 return Register();
2155}
2156
2157Register fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, Register Op0) {
2158 if (RetVT.SimpleTy != MVT::f64)
2159 return Register();
2160 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2161 return fastEmitInst_r(MachineInstOpcode: ARM::VABSD, RC: &ARM::DPRRegClass, Op0);
2162 }
2163 return Register();
2164}
2165
2166Register fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, Register Op0) {
2167 if (RetVT.SimpleTy != MVT::v4f16)
2168 return Register();
2169 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2170 return fastEmitInst_r(MachineInstOpcode: ARM::VABShd, RC: &ARM::DPRRegClass, Op0);
2171 }
2172 return Register();
2173}
2174
2175Register fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, Register Op0) {
2176 if (RetVT.SimpleTy != MVT::v8f16)
2177 return Register();
2178 if ((Subtarget->hasMVEIntegerOps())) {
2179 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSf16, RC: &ARM::MQPRRegClass, Op0);
2180 }
2181 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2182 return fastEmitInst_r(MachineInstOpcode: ARM::VABShq, RC: &ARM::QPRRegClass, Op0);
2183 }
2184 return Register();
2185}
2186
2187Register fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, Register Op0) {
2188 if (RetVT.SimpleTy != MVT::v2f32)
2189 return Register();
2190 if ((Subtarget->hasNEON())) {
2191 return fastEmitInst_r(MachineInstOpcode: ARM::VABSfd, RC: &ARM::DPRRegClass, Op0);
2192 }
2193 return Register();
2194}
2195
2196Register fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, Register Op0) {
2197 if (RetVT.SimpleTy != MVT::v4f32)
2198 return Register();
2199 if ((Subtarget->hasMVEIntegerOps())) {
2200 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSf32, RC: &ARM::MQPRRegClass, Op0);
2201 }
2202 if ((Subtarget->hasNEON())) {
2203 return fastEmitInst_r(MachineInstOpcode: ARM::VABSfq, RC: &ARM::QPRRegClass, Op0);
2204 }
2205 return Register();
2206}
2207
2208Register fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, Register Op0) {
2209 switch (VT.SimpleTy) {
2210 case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0);
2211 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
2212 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
2213 case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0);
2214 case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0);
2215 case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0);
2216 case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0);
2217 default: return Register();
2218 }
2219}
2220
2221// FastEmit functions for ISD::FCEIL.
2222
2223Register fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
2224 if (RetVT.SimpleTy != MVT::f16)
2225 return Register();
2226 if ((Subtarget->hasFullFP16())) {
2227 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPH, RC: &ARM::HPRRegClass, Op0);
2228 }
2229 return Register();
2230}
2231
2232Register fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
2233 if (RetVT.SimpleTy != MVT::f32)
2234 return Register();
2235 if ((Subtarget->hasFPARMv8Base())) {
2236 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPS, RC: &ARM::SPRRegClass, Op0);
2237 }
2238 return Register();
2239}
2240
2241Register fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
2242 if (RetVT.SimpleTy != MVT::f64)
2243 return Register();
2244 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2245 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPD, RC: &ARM::DPRRegClass, Op0);
2246 }
2247 return Register();
2248}
2249
2250Register fastEmit_ISD_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
2251 if (RetVT.SimpleTy != MVT::v4f16)
2252 return Register();
2253 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2254 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNDh, RC: &ARM::DPRRegClass, Op0);
2255 }
2256 return Register();
2257}
2258
2259Register fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
2260 if (RetVT.SimpleTy != MVT::v8f16)
2261 return Register();
2262 if ((Subtarget->hasMVEFloatOps())) {
2263 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16P, RC: &ARM::MQPRRegClass, Op0);
2264 }
2265 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2266 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNQh, RC: &ARM::QPRRegClass, Op0);
2267 }
2268 return Register();
2269}
2270
2271Register fastEmit_ISD_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
2272 if (RetVT.SimpleTy != MVT::v2f32)
2273 return Register();
2274 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2275 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNDf, RC: &ARM::DPRRegClass, Op0);
2276 }
2277 return Register();
2278}
2279
2280Register fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
2281 if (RetVT.SimpleTy != MVT::v4f32)
2282 return Register();
2283 if ((Subtarget->hasMVEFloatOps())) {
2284 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32P, RC: &ARM::MQPRRegClass, Op0);
2285 }
2286 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2287 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNQf, RC: &ARM::QPRRegClass, Op0);
2288 }
2289 return Register();
2290}
2291
2292Register fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
2293 switch (VT.SimpleTy) {
2294 case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0);
2295 case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0);
2296 case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0);
2297 case MVT::v4f16: return fastEmit_ISD_FCEIL_MVT_v4f16_r(RetVT, Op0);
2298 case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0);
2299 case MVT::v2f32: return fastEmit_ISD_FCEIL_MVT_v2f32_r(RetVT, Op0);
2300 case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0);
2301 default: return Register();
2302 }
2303}
2304
2305// FastEmit functions for ISD::FFLOOR.
2306
2307Register fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
2308 if (RetVT.SimpleTy != MVT::f16)
2309 return Register();
2310 if ((Subtarget->hasFullFP16())) {
2311 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMH, RC: &ARM::HPRRegClass, Op0);
2312 }
2313 return Register();
2314}
2315
2316Register fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
2317 if (RetVT.SimpleTy != MVT::f32)
2318 return Register();
2319 if ((Subtarget->hasFPARMv8Base())) {
2320 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMS, RC: &ARM::SPRRegClass, Op0);
2321 }
2322 return Register();
2323}
2324
2325Register fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
2326 if (RetVT.SimpleTy != MVT::f64)
2327 return Register();
2328 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2329 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMD, RC: &ARM::DPRRegClass, Op0);
2330 }
2331 return Register();
2332}
2333
2334Register fastEmit_ISD_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
2335 if (RetVT.SimpleTy != MVT::v4f16)
2336 return Register();
2337 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2338 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNDh, RC: &ARM::DPRRegClass, Op0);
2339 }
2340 return Register();
2341}
2342
2343Register fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
2344 if (RetVT.SimpleTy != MVT::v8f16)
2345 return Register();
2346 if ((Subtarget->hasMVEFloatOps())) {
2347 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16M, RC: &ARM::MQPRRegClass, Op0);
2348 }
2349 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2350 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNQh, RC: &ARM::QPRRegClass, Op0);
2351 }
2352 return Register();
2353}
2354
2355Register fastEmit_ISD_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
2356 if (RetVT.SimpleTy != MVT::v2f32)
2357 return Register();
2358 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2359 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNDf, RC: &ARM::DPRRegClass, Op0);
2360 }
2361 return Register();
2362}
2363
2364Register fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
2365 if (RetVT.SimpleTy != MVT::v4f32)
2366 return Register();
2367 if ((Subtarget->hasMVEFloatOps())) {
2368 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32M, RC: &ARM::MQPRRegClass, Op0);
2369 }
2370 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2371 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNQf, RC: &ARM::QPRRegClass, Op0);
2372 }
2373 return Register();
2374}
2375
2376Register fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
2377 switch (VT.SimpleTy) {
2378 case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0);
2379 case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0);
2380 case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0);
2381 case MVT::v4f16: return fastEmit_ISD_FFLOOR_MVT_v4f16_r(RetVT, Op0);
2382 case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0);
2383 case MVT::v2f32: return fastEmit_ISD_FFLOOR_MVT_v2f32_r(RetVT, Op0);
2384 case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0);
2385 default: return Register();
2386 }
2387}
2388
2389// FastEmit functions for ISD::FNEARBYINT.
2390
2391Register fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
2392 if (RetVT.SimpleTy != MVT::f16)
2393 return Register();
2394 if ((Subtarget->hasFullFP16())) {
2395 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRH, RC: &ARM::HPRRegClass, Op0);
2396 }
2397 return Register();
2398}
2399
2400Register fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
2401 if (RetVT.SimpleTy != MVT::f32)
2402 return Register();
2403 if ((Subtarget->hasFPARMv8Base())) {
2404 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRS, RC: &ARM::SPRRegClass, Op0);
2405 }
2406 return Register();
2407}
2408
2409Register fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
2410 if (RetVT.SimpleTy != MVT::f64)
2411 return Register();
2412 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2413 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRD, RC: &ARM::DPRRegClass, Op0);
2414 }
2415 return Register();
2416}
2417
2418Register fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
2419 switch (VT.SimpleTy) {
2420 case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0);
2421 case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0);
2422 case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0);
2423 default: return Register();
2424 }
2425}
2426
2427// FastEmit functions for ISD::FNEG.
2428
2429Register fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, Register Op0) {
2430 if (RetVT.SimpleTy != MVT::f16)
2431 return Register();
2432 if ((Subtarget->hasFullFP16())) {
2433 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGH, RC: &ARM::HPRRegClass, Op0);
2434 }
2435 return Register();
2436}
2437
2438Register fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, Register Op0) {
2439 if (RetVT.SimpleTy != MVT::f32)
2440 return Register();
2441 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
2442 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGS, RC: &ARM::SPRRegClass, Op0);
2443 }
2444 return Register();
2445}
2446
2447Register fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, Register Op0) {
2448 if (RetVT.SimpleTy != MVT::f64)
2449 return Register();
2450 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2451 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGD, RC: &ARM::DPRRegClass, Op0);
2452 }
2453 return Register();
2454}
2455
2456Register fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, Register Op0) {
2457 if (RetVT.SimpleTy != MVT::v4f16)
2458 return Register();
2459 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2460 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGhd, RC: &ARM::DPRRegClass, Op0);
2461 }
2462 return Register();
2463}
2464
2465Register fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, Register Op0) {
2466 if (RetVT.SimpleTy != MVT::v8f16)
2467 return Register();
2468 if ((Subtarget->hasMVEIntegerOps())) {
2469 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VNEGf16, RC: &ARM::MQPRRegClass, Op0);
2470 }
2471 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2472 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGhq, RC: &ARM::QPRRegClass, Op0);
2473 }
2474 return Register();
2475}
2476
2477Register fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, Register Op0) {
2478 if (RetVT.SimpleTy != MVT::v2f32)
2479 return Register();
2480 if ((Subtarget->hasNEON())) {
2481 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGfd, RC: &ARM::DPRRegClass, Op0);
2482 }
2483 return Register();
2484}
2485
2486Register fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, Register Op0) {
2487 if (RetVT.SimpleTy != MVT::v4f32)
2488 return Register();
2489 if ((Subtarget->hasMVEIntegerOps())) {
2490 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VNEGf32, RC: &ARM::MQPRRegClass, Op0);
2491 }
2492 if ((Subtarget->hasNEON())) {
2493 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGf32q, RC: &ARM::QPRRegClass, Op0);
2494 }
2495 return Register();
2496}
2497
2498Register fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, Register Op0) {
2499 switch (VT.SimpleTy) {
2500 case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0);
2501 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
2502 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
2503 case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0);
2504 case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0);
2505 case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0);
2506 case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0);
2507 default: return Register();
2508 }
2509}
2510
2511// FastEmit functions for ISD::FP_EXTEND.
2512
2513Register fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
2514 if (RetVT.SimpleTy != MVT::f64)
2515 return Register();
2516 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2517 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTDS, RC: &ARM::DPRRegClass, Op0);
2518 }
2519 return Register();
2520}
2521
2522Register fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
2523 if (RetVT.SimpleTy != MVT::v4f32)
2524 return Register();
2525 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2f, RC: &ARM::QPRRegClass, Op0);
2526}
2527
2528Register fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
2529 switch (VT.SimpleTy) {
2530 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
2531 case MVT::v4f16: return fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
2532 default: return Register();
2533 }
2534}
2535
2536// FastEmit functions for ISD::FP_ROUND.
2537
2538Register fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
2539 if (RetVT.SimpleTy != MVT::f32)
2540 return Register();
2541 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2542 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTSD, RC: &ARM::SPRRegClass, Op0);
2543 }
2544 return Register();
2545}
2546
2547Register fastEmit_ISD_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
2548 if (RetVT.SimpleTy != MVT::v4f16)
2549 return Register();
2550 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2h, RC: &ARM::DPRRegClass, Op0);
2551}
2552
2553Register fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
2554 switch (VT.SimpleTy) {
2555 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
2556 case MVT::v4f32: return fastEmit_ISD_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
2557 default: return Register();
2558 }
2559}
2560
2561// FastEmit functions for ISD::FP_TO_SINT.
2562
2563Register fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2564 if (RetVT.SimpleTy != MVT::v4i16)
2565 return Register();
2566 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2567 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2sd, RC: &ARM::DPRRegClass, Op0);
2568 }
2569 return Register();
2570}
2571
2572Register fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2573 if (RetVT.SimpleTy != MVT::v8i16)
2574 return Register();
2575 if ((Subtarget->hasMVEFloatOps())) {
2576 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTs16f16z, RC: &ARM::MQPRRegClass, Op0);
2577 }
2578 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2579 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2sq, RC: &ARM::QPRRegClass, Op0);
2580 }
2581 return Register();
2582}
2583
2584Register fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2585 if (RetVT.SimpleTy != MVT::v2i32)
2586 return Register();
2587 if ((Subtarget->hasNEON())) {
2588 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2sd, RC: &ARM::DPRRegClass, Op0);
2589 }
2590 return Register();
2591}
2592
2593Register fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2594 if (RetVT.SimpleTy != MVT::v4i32)
2595 return Register();
2596 if ((Subtarget->hasMVEFloatOps())) {
2597 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTs32f32z, RC: &ARM::MQPRRegClass, Op0);
2598 }
2599 if ((Subtarget->hasNEON())) {
2600 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2sq, RC: &ARM::QPRRegClass, Op0);
2601 }
2602 return Register();
2603}
2604
2605Register fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
2606 switch (VT.SimpleTy) {
2607 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
2608 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
2609 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
2610 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
2611 default: return Register();
2612 }
2613}
2614
2615// FastEmit functions for ISD::FP_TO_UINT.
2616
2617Register fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2618 if (RetVT.SimpleTy != MVT::v4i16)
2619 return Register();
2620 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2621 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2ud, RC: &ARM::DPRRegClass, Op0);
2622 }
2623 return Register();
2624}
2625
2626Register fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2627 if (RetVT.SimpleTy != MVT::v8i16)
2628 return Register();
2629 if ((Subtarget->hasMVEFloatOps())) {
2630 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTu16f16z, RC: &ARM::MQPRRegClass, Op0);
2631 }
2632 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2633 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2uq, RC: &ARM::QPRRegClass, Op0);
2634 }
2635 return Register();
2636}
2637
2638Register fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2639 if (RetVT.SimpleTy != MVT::v2i32)
2640 return Register();
2641 if ((Subtarget->hasNEON())) {
2642 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2ud, RC: &ARM::DPRRegClass, Op0);
2643 }
2644 return Register();
2645}
2646
2647Register fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2648 if (RetVT.SimpleTy != MVT::v4i32)
2649 return Register();
2650 if ((Subtarget->hasMVEFloatOps())) {
2651 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTu32f32z, RC: &ARM::MQPRRegClass, Op0);
2652 }
2653 if ((Subtarget->hasNEON())) {
2654 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2uq, RC: &ARM::QPRRegClass, Op0);
2655 }
2656 return Register();
2657}
2658
2659Register fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
2660 switch (VT.SimpleTy) {
2661 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
2662 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
2663 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
2664 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
2665 default: return Register();
2666 }
2667}
2668
2669// FastEmit functions for ISD::FRINT.
2670
2671Register fastEmit_ISD_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
2672 if (RetVT.SimpleTy != MVT::f16)
2673 return Register();
2674 if ((Subtarget->hasFullFP16())) {
2675 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXH, RC: &ARM::HPRRegClass, Op0);
2676 }
2677 return Register();
2678}
2679
2680Register fastEmit_ISD_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
2681 if (RetVT.SimpleTy != MVT::f32)
2682 return Register();
2683 if ((Subtarget->hasFPARMv8Base())) {
2684 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXS, RC: &ARM::SPRRegClass, Op0);
2685 }
2686 return Register();
2687}
2688
2689Register fastEmit_ISD_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
2690 if (RetVT.SimpleTy != MVT::f64)
2691 return Register();
2692 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2693 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXD, RC: &ARM::DPRRegClass, Op0);
2694 }
2695 return Register();
2696}
2697
2698Register fastEmit_ISD_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2699 if (RetVT.SimpleTy != MVT::v4f16)
2700 return Register();
2701 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2702 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNDh, RC: &ARM::DPRRegClass, Op0);
2703 }
2704 return Register();
2705}
2706
2707Register fastEmit_ISD_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2708 if (RetVT.SimpleTy != MVT::v8f16)
2709 return Register();
2710 if ((Subtarget->hasMVEFloatOps())) {
2711 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16X, RC: &ARM::MQPRRegClass, Op0);
2712 }
2713 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2714 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNQh, RC: &ARM::QPRRegClass, Op0);
2715 }
2716 return Register();
2717}
2718
2719Register fastEmit_ISD_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2720 if (RetVT.SimpleTy != MVT::v2f32)
2721 return Register();
2722 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2723 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNDf, RC: &ARM::DPRRegClass, Op0);
2724 }
2725 return Register();
2726}
2727
2728Register fastEmit_ISD_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2729 if (RetVT.SimpleTy != MVT::v4f32)
2730 return Register();
2731 if ((Subtarget->hasMVEFloatOps())) {
2732 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32X, RC: &ARM::MQPRRegClass, Op0);
2733 }
2734 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2735 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNQf, RC: &ARM::QPRRegClass, Op0);
2736 }
2737 return Register();
2738}
2739
2740Register fastEmit_ISD_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
2741 switch (VT.SimpleTy) {
2742 case MVT::f16: return fastEmit_ISD_FRINT_MVT_f16_r(RetVT, Op0);
2743 case MVT::f32: return fastEmit_ISD_FRINT_MVT_f32_r(RetVT, Op0);
2744 case MVT::f64: return fastEmit_ISD_FRINT_MVT_f64_r(RetVT, Op0);
2745 case MVT::v4f16: return fastEmit_ISD_FRINT_MVT_v4f16_r(RetVT, Op0);
2746 case MVT::v8f16: return fastEmit_ISD_FRINT_MVT_v8f16_r(RetVT, Op0);
2747 case MVT::v2f32: return fastEmit_ISD_FRINT_MVT_v2f32_r(RetVT, Op0);
2748 case MVT::v4f32: return fastEmit_ISD_FRINT_MVT_v4f32_r(RetVT, Op0);
2749 default: return Register();
2750 }
2751}
2752
2753// FastEmit functions for ISD::FROUND.
2754
2755Register fastEmit_ISD_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
2756 if (RetVT.SimpleTy != MVT::f16)
2757 return Register();
2758 if ((Subtarget->hasFullFP16())) {
2759 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAH, RC: &ARM::HPRRegClass, Op0);
2760 }
2761 return Register();
2762}
2763
2764Register fastEmit_ISD_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
2765 if (RetVT.SimpleTy != MVT::f32)
2766 return Register();
2767 if ((Subtarget->hasFPARMv8Base())) {
2768 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAS, RC: &ARM::SPRRegClass, Op0);
2769 }
2770 return Register();
2771}
2772
2773Register fastEmit_ISD_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
2774 if (RetVT.SimpleTy != MVT::f64)
2775 return Register();
2776 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2777 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAD, RC: &ARM::DPRRegClass, Op0);
2778 }
2779 return Register();
2780}
2781
2782Register fastEmit_ISD_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
2783 if (RetVT.SimpleTy != MVT::v4f16)
2784 return Register();
2785 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2786 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANDh, RC: &ARM::DPRRegClass, Op0);
2787 }
2788 return Register();
2789}
2790
2791Register fastEmit_ISD_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
2792 if (RetVT.SimpleTy != MVT::v8f16)
2793 return Register();
2794 if ((Subtarget->hasMVEFloatOps())) {
2795 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16A, RC: &ARM::MQPRRegClass, Op0);
2796 }
2797 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2798 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANQh, RC: &ARM::QPRRegClass, Op0);
2799 }
2800 return Register();
2801}
2802
2803Register fastEmit_ISD_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
2804 if (RetVT.SimpleTy != MVT::v2f32)
2805 return Register();
2806 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2807 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANDf, RC: &ARM::DPRRegClass, Op0);
2808 }
2809 return Register();
2810}
2811
2812Register fastEmit_ISD_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
2813 if (RetVT.SimpleTy != MVT::v4f32)
2814 return Register();
2815 if ((Subtarget->hasMVEFloatOps())) {
2816 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32A, RC: &ARM::MQPRRegClass, Op0);
2817 }
2818 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2819 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANQf, RC: &ARM::QPRRegClass, Op0);
2820 }
2821 return Register();
2822}
2823
2824Register fastEmit_ISD_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
2825 switch (VT.SimpleTy) {
2826 case MVT::f16: return fastEmit_ISD_FROUND_MVT_f16_r(RetVT, Op0);
2827 case MVT::f32: return fastEmit_ISD_FROUND_MVT_f32_r(RetVT, Op0);
2828 case MVT::f64: return fastEmit_ISD_FROUND_MVT_f64_r(RetVT, Op0);
2829 case MVT::v4f16: return fastEmit_ISD_FROUND_MVT_v4f16_r(RetVT, Op0);
2830 case MVT::v8f16: return fastEmit_ISD_FROUND_MVT_v8f16_r(RetVT, Op0);
2831 case MVT::v2f32: return fastEmit_ISD_FROUND_MVT_v2f32_r(RetVT, Op0);
2832 case MVT::v4f32: return fastEmit_ISD_FROUND_MVT_v4f32_r(RetVT, Op0);
2833 default: return Register();
2834 }
2835}
2836
2837// FastEmit functions for ISD::FROUNDEVEN.
2838
2839Register fastEmit_ISD_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
2840 if (RetVT.SimpleTy != MVT::f16)
2841 return Register();
2842 if ((Subtarget->hasFullFP16())) {
2843 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNH, RC: &ARM::HPRRegClass, Op0);
2844 }
2845 return Register();
2846}
2847
2848Register fastEmit_ISD_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
2849 if (RetVT.SimpleTy != MVT::f32)
2850 return Register();
2851 if ((Subtarget->hasFPARMv8Base())) {
2852 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNS, RC: &ARM::SPRRegClass, Op0);
2853 }
2854 return Register();
2855}
2856
2857Register fastEmit_ISD_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
2858 if (RetVT.SimpleTy != MVT::f64)
2859 return Register();
2860 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2861 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTND, RC: &ARM::DPRRegClass, Op0);
2862 }
2863 return Register();
2864}
2865
2866Register fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
2867 if (RetVT.SimpleTy != MVT::v4f16)
2868 return Register();
2869 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2870 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNDh, RC: &ARM::DPRRegClass, Op0);
2871 }
2872 return Register();
2873}
2874
2875Register fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
2876 if (RetVT.SimpleTy != MVT::v8f16)
2877 return Register();
2878 if ((Subtarget->hasMVEFloatOps())) {
2879 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16N, RC: &ARM::MQPRRegClass, Op0);
2880 }
2881 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2882 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNQh, RC: &ARM::QPRRegClass, Op0);
2883 }
2884 return Register();
2885}
2886
2887Register fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
2888 if (RetVT.SimpleTy != MVT::v2f32)
2889 return Register();
2890 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2891 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNDf, RC: &ARM::DPRRegClass, Op0);
2892 }
2893 return Register();
2894}
2895
2896Register fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
2897 if (RetVT.SimpleTy != MVT::v4f32)
2898 return Register();
2899 if ((Subtarget->hasMVEFloatOps())) {
2900 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32N, RC: &ARM::MQPRRegClass, Op0);
2901 }
2902 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2903 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNQf, RC: &ARM::QPRRegClass, Op0);
2904 }
2905 return Register();
2906}
2907
2908Register fastEmit_ISD_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
2909 switch (VT.SimpleTy) {
2910 case MVT::f16: return fastEmit_ISD_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
2911 case MVT::f32: return fastEmit_ISD_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
2912 case MVT::f64: return fastEmit_ISD_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
2913 case MVT::v4f16: return fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
2914 case MVT::v8f16: return fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
2915 case MVT::v2f32: return fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
2916 case MVT::v4f32: return fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
2917 default: return Register();
2918 }
2919}
2920
2921// FastEmit functions for ISD::FSQRT.
2922
2923Register fastEmit_ISD_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
2924 if (RetVT.SimpleTy != MVT::f16)
2925 return Register();
2926 if ((Subtarget->hasFullFP16())) {
2927 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTH, RC: &ARM::HPRRegClass, Op0);
2928 }
2929 return Register();
2930}
2931
2932Register fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
2933 if (RetVT.SimpleTy != MVT::f32)
2934 return Register();
2935 if ((Subtarget->hasVFP2Base())) {
2936 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTS, RC: &ARM::SPRRegClass, Op0);
2937 }
2938 return Register();
2939}
2940
2941Register fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
2942 if (RetVT.SimpleTy != MVT::f64)
2943 return Register();
2944 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2945 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTD, RC: &ARM::DPRRegClass, Op0);
2946 }
2947 return Register();
2948}
2949
2950Register fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
2951 switch (VT.SimpleTy) {
2952 case MVT::f16: return fastEmit_ISD_FSQRT_MVT_f16_r(RetVT, Op0);
2953 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
2954 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
2955 default: return Register();
2956 }
2957}
2958
2959// FastEmit functions for ISD::FTRUNC.
2960
2961Register fastEmit_ISD_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
2962 if (RetVT.SimpleTy != MVT::f16)
2963 return Register();
2964 if ((Subtarget->hasFullFP16())) {
2965 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZH, RC: &ARM::HPRRegClass, Op0);
2966 }
2967 return Register();
2968}
2969
2970Register fastEmit_ISD_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
2971 if (RetVT.SimpleTy != MVT::f32)
2972 return Register();
2973 if ((Subtarget->hasFPARMv8Base())) {
2974 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZS, RC: &ARM::SPRRegClass, Op0);
2975 }
2976 return Register();
2977}
2978
2979Register fastEmit_ISD_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
2980 if (RetVT.SimpleTy != MVT::f64)
2981 return Register();
2982 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2983 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZD, RC: &ARM::DPRRegClass, Op0);
2984 }
2985 return Register();
2986}
2987
2988Register fastEmit_ISD_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
2989 if (RetVT.SimpleTy != MVT::v4f16)
2990 return Register();
2991 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2992 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNDh, RC: &ARM::DPRRegClass, Op0);
2993 }
2994 return Register();
2995}
2996
2997Register fastEmit_ISD_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
2998 if (RetVT.SimpleTy != MVT::v8f16)
2999 return Register();
3000 if ((Subtarget->hasMVEFloatOps())) {
3001 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16Z, RC: &ARM::MQPRRegClass, Op0);
3002 }
3003 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3004 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNQh, RC: &ARM::QPRRegClass, Op0);
3005 }
3006 return Register();
3007}
3008
3009Register fastEmit_ISD_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
3010 if (RetVT.SimpleTy != MVT::v2f32)
3011 return Register();
3012 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3013 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNDf, RC: &ARM::DPRRegClass, Op0);
3014 }
3015 return Register();
3016}
3017
3018Register fastEmit_ISD_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
3019 if (RetVT.SimpleTy != MVT::v4f32)
3020 return Register();
3021 if ((Subtarget->hasMVEFloatOps())) {
3022 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32Z, RC: &ARM::MQPRRegClass, Op0);
3023 }
3024 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3025 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNQf, RC: &ARM::QPRRegClass, Op0);
3026 }
3027 return Register();
3028}
3029
3030Register fastEmit_ISD_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
3031 switch (VT.SimpleTy) {
3032 case MVT::f16: return fastEmit_ISD_FTRUNC_MVT_f16_r(RetVT, Op0);
3033 case MVT::f32: return fastEmit_ISD_FTRUNC_MVT_f32_r(RetVT, Op0);
3034 case MVT::f64: return fastEmit_ISD_FTRUNC_MVT_f64_r(RetVT, Op0);
3035 case MVT::v4f16: return fastEmit_ISD_FTRUNC_MVT_v4f16_r(RetVT, Op0);
3036 case MVT::v8f16: return fastEmit_ISD_FTRUNC_MVT_v8f16_r(RetVT, Op0);
3037 case MVT::v2f32: return fastEmit_ISD_FTRUNC_MVT_v2f32_r(RetVT, Op0);
3038 case MVT::v4f32: return fastEmit_ISD_FTRUNC_MVT_v4f32_r(RetVT, Op0);
3039 default: return Register();
3040 }
3041}
3042
3043// FastEmit functions for ISD::SET_FPENV.
3044
3045Register fastEmit_ISD_SET_FPENV_MVT_i32_r(MVT RetVT, Register Op0) {
3046 if (RetVT.SimpleTy != MVT::isVoid)
3047 return Register();
3048 return fastEmitInst_r(MachineInstOpcode: ARM::VMSR, RC: &ARM::GPRnopcRegClass, Op0);
3049}
3050
3051Register fastEmit_ISD_SET_FPENV_r(MVT VT, MVT RetVT, Register Op0) {
3052 switch (VT.SimpleTy) {
3053 case MVT::i32: return fastEmit_ISD_SET_FPENV_MVT_i32_r(RetVT, Op0);
3054 default: return Register();
3055 }
3056}
3057
3058// FastEmit functions for ISD::SIGN_EXTEND.
3059
3060Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i8_r(MVT RetVT, Register Op0) {
3061 if (RetVT.SimpleTy != MVT::v8i16)
3062 return Register();
3063 if ((Subtarget->hasNEON())) {
3064 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLsv8i16, RC: &ARM::QPRRegClass, Op0);
3065 }
3066 return Register();
3067}
3068
3069Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i16_r(MVT RetVT, Register Op0) {
3070 if (RetVT.SimpleTy != MVT::v4i32)
3071 return Register();
3072 if ((Subtarget->hasNEON())) {
3073 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLsv4i32, RC: &ARM::QPRRegClass, Op0);
3074 }
3075 return Register();
3076}
3077
3078Register fastEmit_ISD_SIGN_EXTEND_MVT_v2i32_r(MVT RetVT, Register Op0) {
3079 if (RetVT.SimpleTy != MVT::v2i64)
3080 return Register();
3081 if ((Subtarget->hasNEON())) {
3082 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLsv2i64, RC: &ARM::QPRRegClass, Op0);
3083 }
3084 return Register();
3085}
3086
3087Register fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3088 switch (VT.SimpleTy) {
3089 case MVT::v8i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i8_r(RetVT, Op0);
3090 case MVT::v4i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i16_r(RetVT, Op0);
3091 case MVT::v2i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i32_r(RetVT, Op0);
3092 default: return Register();
3093 }
3094}
3095
3096// FastEmit functions for ISD::SINT_TO_FP.
3097
3098Register fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
3099 if (RetVT.SimpleTy != MVT::v4f16)
3100 return Register();
3101 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3102 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2hd, RC: &ARM::DPRRegClass, Op0);
3103 }
3104 return Register();
3105}
3106
3107Register fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
3108 if (RetVT.SimpleTy != MVT::v8f16)
3109 return Register();
3110 if ((Subtarget->hasMVEFloatOps())) {
3111 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf16s16n, RC: &ARM::MQPRRegClass, Op0);
3112 }
3113 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3114 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2hq, RC: &ARM::QPRRegClass, Op0);
3115 }
3116 return Register();
3117}
3118
3119Register fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
3120 if (RetVT.SimpleTy != MVT::v2f32)
3121 return Register();
3122 if ((Subtarget->hasNEON())) {
3123 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2fd, RC: &ARM::DPRRegClass, Op0);
3124 }
3125 return Register();
3126}
3127
3128Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
3129 if (RetVT.SimpleTy != MVT::v4f32)
3130 return Register();
3131 if ((Subtarget->hasMVEFloatOps())) {
3132 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf32s32n, RC: &ARM::MQPRRegClass, Op0);
3133 }
3134 if ((Subtarget->hasNEON())) {
3135 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2fq, RC: &ARM::QPRRegClass, Op0);
3136 }
3137 return Register();
3138}
3139
3140Register fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3141 switch (VT.SimpleTy) {
3142 case MVT::v4i16: return fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
3143 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3144 case MVT::v2i32: return fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
3145 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3146 default: return Register();
3147 }
3148}
3149
3150// FastEmit functions for ISD::STRICT_FCEIL.
3151
3152Register fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
3153 if (RetVT.SimpleTy != MVT::f16)
3154 return Register();
3155 if ((Subtarget->hasFullFP16())) {
3156 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPH, RC: &ARM::HPRRegClass, Op0);
3157 }
3158 return Register();
3159}
3160
3161Register fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
3162 if (RetVT.SimpleTy != MVT::f32)
3163 return Register();
3164 if ((Subtarget->hasFPARMv8Base())) {
3165 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPS, RC: &ARM::SPRRegClass, Op0);
3166 }
3167 return Register();
3168}
3169
3170Register fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
3171 if (RetVT.SimpleTy != MVT::f64)
3172 return Register();
3173 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3174 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPD, RC: &ARM::DPRRegClass, Op0);
3175 }
3176 return Register();
3177}
3178
3179Register fastEmit_ISD_STRICT_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
3180 switch (VT.SimpleTy) {
3181 case MVT::f16: return fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(RetVT, Op0);
3182 case MVT::f32: return fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(RetVT, Op0);
3183 case MVT::f64: return fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(RetVT, Op0);
3184 default: return Register();
3185 }
3186}
3187
3188// FastEmit functions for ISD::STRICT_FFLOOR.
3189
3190Register fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
3191 if (RetVT.SimpleTy != MVT::f16)
3192 return Register();
3193 if ((Subtarget->hasFullFP16())) {
3194 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMH, RC: &ARM::HPRRegClass, Op0);
3195 }
3196 return Register();
3197}
3198
3199Register fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
3200 if (RetVT.SimpleTy != MVT::f32)
3201 return Register();
3202 if ((Subtarget->hasFPARMv8Base())) {
3203 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMS, RC: &ARM::SPRRegClass, Op0);
3204 }
3205 return Register();
3206}
3207
3208Register fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
3209 if (RetVT.SimpleTy != MVT::f64)
3210 return Register();
3211 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3212 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMD, RC: &ARM::DPRRegClass, Op0);
3213 }
3214 return Register();
3215}
3216
3217Register fastEmit_ISD_STRICT_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
3218 switch (VT.SimpleTy) {
3219 case MVT::f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(RetVT, Op0);
3220 case MVT::f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(RetVT, Op0);
3221 case MVT::f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(RetVT, Op0);
3222 default: return Register();
3223 }
3224}
3225
3226// FastEmit functions for ISD::STRICT_FNEARBYINT.
3227
3228Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
3229 if (RetVT.SimpleTy != MVT::f16)
3230 return Register();
3231 if ((Subtarget->hasFullFP16())) {
3232 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRH, RC: &ARM::HPRRegClass, Op0);
3233 }
3234 return Register();
3235}
3236
3237Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
3238 if (RetVT.SimpleTy != MVT::f32)
3239 return Register();
3240 if ((Subtarget->hasFPARMv8Base())) {
3241 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRS, RC: &ARM::SPRRegClass, Op0);
3242 }
3243 return Register();
3244}
3245
3246Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
3247 if (RetVT.SimpleTy != MVT::f64)
3248 return Register();
3249 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3250 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRD, RC: &ARM::DPRRegClass, Op0);
3251 }
3252 return Register();
3253}
3254
3255Register fastEmit_ISD_STRICT_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
3256 switch (VT.SimpleTy) {
3257 case MVT::f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(RetVT, Op0);
3258 case MVT::f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(RetVT, Op0);
3259 case MVT::f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(RetVT, Op0);
3260 default: return Register();
3261 }
3262}
3263
3264// FastEmit functions for ISD::STRICT_FP_EXTEND.
3265
3266Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
3267 if (RetVT.SimpleTy != MVT::f64)
3268 return Register();
3269 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3270 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTDS, RC: &ARM::DPRRegClass, Op0);
3271 }
3272 return Register();
3273}
3274
3275Register fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3276 switch (VT.SimpleTy) {
3277 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0);
3278 default: return Register();
3279 }
3280}
3281
3282// FastEmit functions for ISD::STRICT_FP_ROUND.
3283
3284Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3285 if (RetVT.SimpleTy != MVT::f32)
3286 return Register();
3287 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3288 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTSD, RC: &ARM::SPRRegClass, Op0);
3289 }
3290 return Register();
3291}
3292
3293Register fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
3294 switch (VT.SimpleTy) {
3295 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0);
3296 default: return Register();
3297 }
3298}
3299
3300// FastEmit functions for ISD::STRICT_FRINT.
3301
3302Register fastEmit_ISD_STRICT_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
3303 if (RetVT.SimpleTy != MVT::f16)
3304 return Register();
3305 if ((Subtarget->hasFullFP16())) {
3306 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXH, RC: &ARM::HPRRegClass, Op0);
3307 }
3308 return Register();
3309}
3310
3311Register fastEmit_ISD_STRICT_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
3312 if (RetVT.SimpleTy != MVT::f32)
3313 return Register();
3314 if ((Subtarget->hasFPARMv8Base())) {
3315 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXS, RC: &ARM::SPRRegClass, Op0);
3316 }
3317 return Register();
3318}
3319
3320Register fastEmit_ISD_STRICT_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
3321 if (RetVT.SimpleTy != MVT::f64)
3322 return Register();
3323 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3324 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXD, RC: &ARM::DPRRegClass, Op0);
3325 }
3326 return Register();
3327}
3328
3329Register fastEmit_ISD_STRICT_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
3330 switch (VT.SimpleTy) {
3331 case MVT::f16: return fastEmit_ISD_STRICT_FRINT_MVT_f16_r(RetVT, Op0);
3332 case MVT::f32: return fastEmit_ISD_STRICT_FRINT_MVT_f32_r(RetVT, Op0);
3333 case MVT::f64: return fastEmit_ISD_STRICT_FRINT_MVT_f64_r(RetVT, Op0);
3334 default: return Register();
3335 }
3336}
3337
3338// FastEmit functions for ISD::STRICT_FROUND.
3339
3340Register fastEmit_ISD_STRICT_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
3341 if (RetVT.SimpleTy != MVT::f16)
3342 return Register();
3343 if ((Subtarget->hasFullFP16())) {
3344 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAH, RC: &ARM::HPRRegClass, Op0);
3345 }
3346 return Register();
3347}
3348
3349Register fastEmit_ISD_STRICT_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3350 if (RetVT.SimpleTy != MVT::f32)
3351 return Register();
3352 if ((Subtarget->hasFPARMv8Base())) {
3353 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAS, RC: &ARM::SPRRegClass, Op0);
3354 }
3355 return Register();
3356}
3357
3358Register fastEmit_ISD_STRICT_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3359 if (RetVT.SimpleTy != MVT::f64)
3360 return Register();
3361 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3362 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAD, RC: &ARM::DPRRegClass, Op0);
3363 }
3364 return Register();
3365}
3366
3367Register fastEmit_ISD_STRICT_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
3368 switch (VT.SimpleTy) {
3369 case MVT::f16: return fastEmit_ISD_STRICT_FROUND_MVT_f16_r(RetVT, Op0);
3370 case MVT::f32: return fastEmit_ISD_STRICT_FROUND_MVT_f32_r(RetVT, Op0);
3371 case MVT::f64: return fastEmit_ISD_STRICT_FROUND_MVT_f64_r(RetVT, Op0);
3372 default: return Register();
3373 }
3374}
3375
3376// FastEmit functions for ISD::STRICT_FROUNDEVEN.
3377
3378Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
3379 if (RetVT.SimpleTy != MVT::f16)
3380 return Register();
3381 if ((Subtarget->hasFullFP16())) {
3382 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNH, RC: &ARM::HPRRegClass, Op0);
3383 }
3384 return Register();
3385}
3386
3387Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
3388 if (RetVT.SimpleTy != MVT::f32)
3389 return Register();
3390 if ((Subtarget->hasFPARMv8Base())) {
3391 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNS, RC: &ARM::SPRRegClass, Op0);
3392 }
3393 return Register();
3394}
3395
3396Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
3397 if (RetVT.SimpleTy != MVT::f64)
3398 return Register();
3399 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3400 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTND, RC: &ARM::DPRRegClass, Op0);
3401 }
3402 return Register();
3403}
3404
3405Register fastEmit_ISD_STRICT_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
3406 switch (VT.SimpleTy) {
3407 case MVT::f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
3408 case MVT::f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
3409 case MVT::f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
3410 default: return Register();
3411 }
3412}
3413
3414// FastEmit functions for ISD::STRICT_FSQRT.
3415
3416Register fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
3417 if (RetVT.SimpleTy != MVT::f16)
3418 return Register();
3419 if ((Subtarget->hasFullFP16())) {
3420 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTH, RC: &ARM::HPRRegClass, Op0);
3421 }
3422 return Register();
3423}
3424
3425Register fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
3426 if (RetVT.SimpleTy != MVT::f32)
3427 return Register();
3428 if ((Subtarget->hasVFP2Base())) {
3429 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTS, RC: &ARM::SPRRegClass, Op0);
3430 }
3431 return Register();
3432}
3433
3434Register fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
3435 if (RetVT.SimpleTy != MVT::f64)
3436 return Register();
3437 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3438 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTD, RC: &ARM::DPRRegClass, Op0);
3439 }
3440 return Register();
3441}
3442
3443Register fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
3444 switch (VT.SimpleTy) {
3445 case MVT::f16: return fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(RetVT, Op0);
3446 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0);
3447 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0);
3448 default: return Register();
3449 }
3450}
3451
3452// FastEmit functions for ISD::STRICT_FTRUNC.
3453
3454Register fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
3455 if (RetVT.SimpleTy != MVT::f16)
3456 return Register();
3457 if ((Subtarget->hasFullFP16())) {
3458 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZH, RC: &ARM::HPRRegClass, Op0);
3459 }
3460 return Register();
3461}
3462
3463Register fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
3464 if (RetVT.SimpleTy != MVT::f32)
3465 return Register();
3466 if ((Subtarget->hasFPARMv8Base())) {
3467 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZS, RC: &ARM::SPRRegClass, Op0);
3468 }
3469 return Register();
3470}
3471
3472Register fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
3473 if (RetVT.SimpleTy != MVT::f64)
3474 return Register();
3475 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3476 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZD, RC: &ARM::DPRRegClass, Op0);
3477 }
3478 return Register();
3479}
3480
3481Register fastEmit_ISD_STRICT_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
3482 switch (VT.SimpleTy) {
3483 case MVT::f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(RetVT, Op0);
3484 case MVT::f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(RetVT, Op0);
3485 case MVT::f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(RetVT, Op0);
3486 default: return Register();
3487 }
3488}
3489
3490// FastEmit functions for ISD::TRUNCATE.
3491
3492Register fastEmit_ISD_TRUNCATE_MVT_v8i16_r(MVT RetVT, Register Op0) {
3493 if (RetVT.SimpleTy != MVT::v8i8)
3494 return Register();
3495 if ((Subtarget->hasNEON())) {
3496 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVNv8i8, RC: &ARM::DPRRegClass, Op0);
3497 }
3498 return Register();
3499}
3500
3501Register fastEmit_ISD_TRUNCATE_MVT_v4i32_r(MVT RetVT, Register Op0) {
3502 if (RetVT.SimpleTy != MVT::v4i16)
3503 return Register();
3504 if ((Subtarget->hasNEON())) {
3505 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVNv4i16, RC: &ARM::DPRRegClass, Op0);
3506 }
3507 return Register();
3508}
3509
3510Register fastEmit_ISD_TRUNCATE_MVT_v2i64_r(MVT RetVT, Register Op0) {
3511 if (RetVT.SimpleTy != MVT::v2i32)
3512 return Register();
3513 if ((Subtarget->hasNEON())) {
3514 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVNv2i32, RC: &ARM::DPRRegClass, Op0);
3515 }
3516 return Register();
3517}
3518
3519Register fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, Register Op0) {
3520 switch (VT.SimpleTy) {
3521 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i16_r(RetVT, Op0);
3522 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_MVT_v4i32_r(RetVT, Op0);
3523 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_MVT_v2i64_r(RetVT, Op0);
3524 default: return Register();
3525 }
3526}
3527
3528// FastEmit functions for ISD::UINT_TO_FP.
3529
3530Register fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
3531 if (RetVT.SimpleTy != MVT::v4f16)
3532 return Register();
3533 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3534 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2hd, RC: &ARM::DPRRegClass, Op0);
3535 }
3536 return Register();
3537}
3538
3539Register fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
3540 if (RetVT.SimpleTy != MVT::v8f16)
3541 return Register();
3542 if ((Subtarget->hasMVEFloatOps())) {
3543 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf16u16n, RC: &ARM::MQPRRegClass, Op0);
3544 }
3545 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3546 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2hq, RC: &ARM::QPRRegClass, Op0);
3547 }
3548 return Register();
3549}
3550
3551Register fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
3552 if (RetVT.SimpleTy != MVT::v2f32)
3553 return Register();
3554 if ((Subtarget->hasNEON())) {
3555 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2fd, RC: &ARM::DPRRegClass, Op0);
3556 }
3557 return Register();
3558}
3559
3560Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
3561 if (RetVT.SimpleTy != MVT::v4f32)
3562 return Register();
3563 if ((Subtarget->hasMVEFloatOps())) {
3564 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf32u32n, RC: &ARM::MQPRRegClass, Op0);
3565 }
3566 if ((Subtarget->hasNEON())) {
3567 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2fq, RC: &ARM::QPRRegClass, Op0);
3568 }
3569 return Register();
3570}
3571
3572Register fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3573 switch (VT.SimpleTy) {
3574 case MVT::v4i16: return fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
3575 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3576 case MVT::v2i32: return fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
3577 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3578 default: return Register();
3579 }
3580}
3581
3582// FastEmit functions for ISD::VECREDUCE_ADD.
3583
3584Register fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(MVT RetVT, Register Op0) {
3585 if (RetVT.SimpleTy != MVT::i32)
3586 return Register();
3587 if ((Subtarget->hasMVEIntegerOps())) {
3588 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu8no_acc, RC: &ARM::tGPREvenRegClass, Op0);
3589 }
3590 return Register();
3591}
3592
3593Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(MVT RetVT, Register Op0) {
3594 if (RetVT.SimpleTy != MVT::i32)
3595 return Register();
3596 if ((Subtarget->hasMVEIntegerOps())) {
3597 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu16no_acc, RC: &ARM::tGPREvenRegClass, Op0);
3598 }
3599 return Register();
3600}
3601
3602Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(MVT RetVT, Register Op0) {
3603 if (RetVT.SimpleTy != MVT::i32)
3604 return Register();
3605 if ((Subtarget->hasMVEIntegerOps())) {
3606 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu32no_acc, RC: &ARM::tGPREvenRegClass, Op0);
3607 }
3608 return Register();
3609}
3610
3611Register fastEmit_ISD_VECREDUCE_ADD_r(MVT VT, MVT RetVT, Register Op0) {
3612 switch (VT.SimpleTy) {
3613 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(RetVT, Op0);
3614 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(RetVT, Op0);
3615 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(RetVT, Op0);
3616 default: return Register();
3617 }
3618}
3619
3620// FastEmit functions for ISD::ZERO_EXTEND.
3621
3622Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i8_r(MVT RetVT, Register Op0) {
3623 if (RetVT.SimpleTy != MVT::v8i16)
3624 return Register();
3625 if ((Subtarget->hasNEON())) {
3626 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv8i16, RC: &ARM::QPRRegClass, Op0);
3627 }
3628 return Register();
3629}
3630
3631Register fastEmit_ISD_ZERO_EXTEND_MVT_v4i16_r(MVT RetVT, Register Op0) {
3632 if (RetVT.SimpleTy != MVT::v4i32)
3633 return Register();
3634 if ((Subtarget->hasNEON())) {
3635 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv4i32, RC: &ARM::QPRRegClass, Op0);
3636 }
3637 return Register();
3638}
3639
3640Register fastEmit_ISD_ZERO_EXTEND_MVT_v2i32_r(MVT RetVT, Register Op0) {
3641 if (RetVT.SimpleTy != MVT::v2i64)
3642 return Register();
3643 if ((Subtarget->hasNEON())) {
3644 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv2i64, RC: &ARM::QPRRegClass, Op0);
3645 }
3646 return Register();
3647}
3648
3649Register fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3650 switch (VT.SimpleTy) {
3651 case MVT::v8i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i8_r(RetVT, Op0);
3652 case MVT::v4i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v4i16_r(RetVT, Op0);
3653 case MVT::v2i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v2i32_r(RetVT, Op0);
3654 default: return Register();
3655 }
3656}
3657
3658// Top-level FastEmit function.
3659
3660Register fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, Register Op0) override {
3661 switch (Opcode) {
3662 case ARMISD::CALL: return fastEmit_ARMISD_CALL_r(VT, RetVT, Op0);
3663 case ARMISD::CALL_NOLINK: return fastEmit_ARMISD_CALL_NOLINK_r(VT, RetVT, Op0);
3664 case ARMISD::CALL_PRED: return fastEmit_ARMISD_CALL_PRED_r(VT, RetVT, Op0);
3665 case ARMISD::CMPFPEw0: return fastEmit_ARMISD_CMPFPEw0_r(VT, RetVT, Op0);
3666 case ARMISD::CMPFPw0: return fastEmit_ARMISD_CMPFPw0_r(VT, RetVT, Op0);
3667 case ARMISD::VADDVs: return fastEmit_ARMISD_VADDVs_r(VT, RetVT, Op0);
3668 case ARMISD::VADDVu: return fastEmit_ARMISD_VADDVu_r(VT, RetVT, Op0);
3669 case ARMISD::VDUP: return fastEmit_ARMISD_VDUP_r(VT, RetVT, Op0);
3670 case ARMISD::VMOVSR: return fastEmit_ARMISD_VMOVSR_r(VT, RetVT, Op0);
3671 case ARMISD::VMOVhr: return fastEmit_ARMISD_VMOVhr_r(VT, RetVT, Op0);
3672 case ARMISD::VMOVrh: return fastEmit_ARMISD_VMOVrh_r(VT, RetVT, Op0);
3673 case ARMISD::VREV16: return fastEmit_ARMISD_VREV16_r(VT, RetVT, Op0);
3674 case ARMISD::VREV32: return fastEmit_ARMISD_VREV32_r(VT, RetVT, Op0);
3675 case ARMISD::VREV64: return fastEmit_ARMISD_VREV64_r(VT, RetVT, Op0);
3676 case ARMISD::WIN__DBZCHK: return fastEmit_ARMISD_WIN__DBZCHK_r(VT, RetVT, Op0);
3677 case ARMISD::WLSSETUP: return fastEmit_ARMISD_WLSSETUP_r(VT, RetVT, Op0);
3678 case ARMISD::tSECALL: return fastEmit_ARMISD_tSECALL_r(VT, RetVT, Op0);
3679 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
3680 case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0);
3681 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
3682 case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0);
3683 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0);
3684 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
3685 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
3686 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
3687 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
3688 case ISD::FCEIL: return fastEmit_ISD_FCEIL_r(VT, RetVT, Op0);
3689 case ISD::FFLOOR: return fastEmit_ISD_FFLOOR_r(VT, RetVT, Op0);
3690 case ISD::FNEARBYINT: return fastEmit_ISD_FNEARBYINT_r(VT, RetVT, Op0);
3691 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
3692 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
3693 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
3694 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
3695 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
3696 case ISD::FRINT: return fastEmit_ISD_FRINT_r(VT, RetVT, Op0);
3697 case ISD::FROUND: return fastEmit_ISD_FROUND_r(VT, RetVT, Op0);
3698 case ISD::FROUNDEVEN: return fastEmit_ISD_FROUNDEVEN_r(VT, RetVT, Op0);
3699 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
3700 case ISD::FTRUNC: return fastEmit_ISD_FTRUNC_r(VT, RetVT, Op0);
3701 case ISD::SET_FPENV: return fastEmit_ISD_SET_FPENV_r(VT, RetVT, Op0);
3702 case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0);
3703 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
3704 case ISD::STRICT_FCEIL: return fastEmit_ISD_STRICT_FCEIL_r(VT, RetVT, Op0);
3705 case ISD::STRICT_FFLOOR: return fastEmit_ISD_STRICT_FFLOOR_r(VT, RetVT, Op0);
3706 case ISD::STRICT_FNEARBYINT: return fastEmit_ISD_STRICT_FNEARBYINT_r(VT, RetVT, Op0);
3707 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0);
3708 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0);
3709 case ISD::STRICT_FRINT: return fastEmit_ISD_STRICT_FRINT_r(VT, RetVT, Op0);
3710 case ISD::STRICT_FROUND: return fastEmit_ISD_STRICT_FROUND_r(VT, RetVT, Op0);
3711 case ISD::STRICT_FROUNDEVEN: return fastEmit_ISD_STRICT_FROUNDEVEN_r(VT, RetVT, Op0);
3712 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0);
3713 case ISD::STRICT_FTRUNC: return fastEmit_ISD_STRICT_FTRUNC_r(VT, RetVT, Op0);
3714 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
3715 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
3716 case ISD::VECREDUCE_ADD: return fastEmit_ISD_VECREDUCE_ADD_r(VT, RetVT, Op0);
3717 case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0);
3718 default: return Register();
3719 }
3720}
3721
3722// FastEmit functions for ARMISD::CMP.
3723
3724Register fastEmit_ARMISD_CMP_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3725 if (RetVT.SimpleTy != MVT::i32)
3726 return Register();
3727 if ((Subtarget->isThumb2())) {
3728 return fastEmitInst_rr(MachineInstOpcode: ARM::t2CMPrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3729 }
3730 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
3731 return fastEmitInst_rr(MachineInstOpcode: ARM::tCMPr, RC: &ARM::tGPRRegClass, Op0, Op1);
3732 }
3733 if ((!Subtarget->isThumb())) {
3734 return fastEmitInst_rr(MachineInstOpcode: ARM::CMPrr, RC: &ARM::GPRRegClass, Op0, Op1);
3735 }
3736 return Register();
3737}
3738
3739Register fastEmit_ARMISD_CMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3740 switch (VT.SimpleTy) {
3741 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_rr(RetVT, Op0, Op1);
3742 default: return Register();
3743 }
3744}
3745
3746// FastEmit functions for ARMISD::CMPFP.
3747
3748Register fastEmit_ARMISD_CMPFP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
3749 if (RetVT.SimpleTy != MVT::i32)
3750 return Register();
3751 if ((Subtarget->hasFullFP16())) {
3752 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPH, RC: &ARM::HPRRegClass, Op0, Op1);
3753 }
3754 return Register();
3755}
3756
3757Register fastEmit_ARMISD_CMPFP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
3758 if (RetVT.SimpleTy != MVT::i32)
3759 return Register();
3760 if ((Subtarget->hasVFP2Base())) {
3761 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPS, RC: &ARM::SPRRegClass, Op0, Op1);
3762 }
3763 return Register();
3764}
3765
3766Register fastEmit_ARMISD_CMPFP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
3767 if (RetVT.SimpleTy != MVT::i32)
3768 return Register();
3769 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3770 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPD, RC: &ARM::DPRRegClass, Op0, Op1);
3771 }
3772 return Register();
3773}
3774
3775Register fastEmit_ARMISD_CMPFP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3776 switch (VT.SimpleTy) {
3777 case MVT::f16: return fastEmit_ARMISD_CMPFP_MVT_f16_rr(RetVT, Op0, Op1);
3778 case MVT::f32: return fastEmit_ARMISD_CMPFP_MVT_f32_rr(RetVT, Op0, Op1);
3779 case MVT::f64: return fastEmit_ARMISD_CMPFP_MVT_f64_rr(RetVT, Op0, Op1);
3780 default: return Register();
3781 }
3782}
3783
3784// FastEmit functions for ARMISD::CMPFPE.
3785
3786Register fastEmit_ARMISD_CMPFPE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
3787 if (RetVT.SimpleTy != MVT::i32)
3788 return Register();
3789 if ((Subtarget->hasFullFP16())) {
3790 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPEH, RC: &ARM::HPRRegClass, Op0, Op1);
3791 }
3792 return Register();
3793}
3794
3795Register fastEmit_ARMISD_CMPFPE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
3796 if (RetVT.SimpleTy != MVT::i32)
3797 return Register();
3798 if ((Subtarget->hasVFP2Base())) {
3799 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPES, RC: &ARM::SPRRegClass, Op0, Op1);
3800 }
3801 return Register();
3802}
3803
3804Register fastEmit_ARMISD_CMPFPE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
3805 if (RetVT.SimpleTy != MVT::i32)
3806 return Register();
3807 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3808 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPED, RC: &ARM::DPRRegClass, Op0, Op1);
3809 }
3810 return Register();
3811}
3812
3813Register fastEmit_ARMISD_CMPFPE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3814 switch (VT.SimpleTy) {
3815 case MVT::f16: return fastEmit_ARMISD_CMPFPE_MVT_f16_rr(RetVT, Op0, Op1);
3816 case MVT::f32: return fastEmit_ARMISD_CMPFPE_MVT_f32_rr(RetVT, Op0, Op1);
3817 case MVT::f64: return fastEmit_ARMISD_CMPFPE_MVT_f64_rr(RetVT, Op0, Op1);
3818 default: return Register();
3819 }
3820}
3821
3822// FastEmit functions for ARMISD::CMPZ.
3823
3824Register fastEmit_ARMISD_CMPZ_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3825 if (RetVT.SimpleTy != MVT::i32)
3826 return Register();
3827 if ((Subtarget->isThumb2())) {
3828 return fastEmitInst_rr(MachineInstOpcode: ARM::t2CMPrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3829 }
3830 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
3831 return fastEmitInst_rr(MachineInstOpcode: ARM::tCMPr, RC: &ARM::tGPRRegClass, Op0, Op1);
3832 }
3833 if ((!Subtarget->isThumb())) {
3834 return fastEmitInst_rr(MachineInstOpcode: ARM::CMPrr, RC: &ARM::GPRRegClass, Op0, Op1);
3835 }
3836 return Register();
3837}
3838
3839Register fastEmit_ARMISD_CMPZ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3840 switch (VT.SimpleTy) {
3841 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_rr(RetVT, Op0, Op1);
3842 default: return Register();
3843 }
3844}
3845
3846// FastEmit functions for ARMISD::EH_SJLJ_LONGJMP.
3847
3848Register fastEmit_ARMISD_EH_SJLJ_LONGJMP_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3849 if (RetVT.SimpleTy != MVT::isVoid)
3850 return Register();
3851 if ((Subtarget->isThumb()) && (Subtarget->isTargetWindows())) {
3852 return fastEmitInst_rr(MachineInstOpcode: ARM::tInt_WIN_eh_sjlj_longjmp, RC: &ARM::GPRRegClass, Op0, Op1);
3853 }
3854 if ((!Subtarget->isTargetWindows()) && (Subtarget->isThumb())) {
3855 return fastEmitInst_rr(MachineInstOpcode: ARM::tInt_eh_sjlj_longjmp, RC: &ARM::tGPRRegClass, Op0, Op1);
3856 }
3857 if ((!Subtarget->isThumb())) {
3858 return fastEmitInst_rr(MachineInstOpcode: ARM::Int_eh_sjlj_longjmp, RC: &ARM::GPRRegClass, Op0, Op1);
3859 }
3860 return Register();
3861}
3862
3863Register fastEmit_ARMISD_EH_SJLJ_LONGJMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3864 switch (VT.SimpleTy) {
3865 case MVT::i32: return fastEmit_ARMISD_EH_SJLJ_LONGJMP_MVT_i32_rr(RetVT, Op0, Op1);
3866 default: return Register();
3867 }
3868}
3869
3870// FastEmit functions for ARMISD::EH_SJLJ_SETJMP.
3871
3872Register fastEmit_ARMISD_EH_SJLJ_SETJMP_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3873 if (RetVT.SimpleTy != MVT::i32)
3874 return Register();
3875 if ((Subtarget->isThumb2()) && (!Subtarget->hasVFP2Base())) {
3876 return fastEmitInst_rr(MachineInstOpcode: ARM::t2Int_eh_sjlj_setjmp_nofp, RC: &ARM::tGPRRegClass, Op0, Op1);
3877 }
3878 if ((Subtarget->hasVFP2Base()) && (Subtarget->isThumb2())) {
3879 return fastEmitInst_rr(MachineInstOpcode: ARM::t2Int_eh_sjlj_setjmp, RC: &ARM::tGPRRegClass, Op0, Op1);
3880 }
3881 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
3882 return fastEmitInst_rr(MachineInstOpcode: ARM::tInt_eh_sjlj_setjmp, RC: &ARM::tGPRRegClass, Op0, Op1);
3883 }
3884 if ((!Subtarget->isThumb()) && (!Subtarget->hasVFP2Base())) {
3885 return fastEmitInst_rr(MachineInstOpcode: ARM::Int_eh_sjlj_setjmp_nofp, RC: &ARM::GPRRegClass, Op0, Op1);
3886 }
3887 if ((Subtarget->hasVFP2Base()) && (!Subtarget->isThumb())) {
3888 return fastEmitInst_rr(MachineInstOpcode: ARM::Int_eh_sjlj_setjmp, RC: &ARM::GPRRegClass, Op0, Op1);
3889 }
3890 return Register();
3891}
3892
3893Register fastEmit_ARMISD_EH_SJLJ_SETJMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3894 switch (VT.SimpleTy) {
3895 case MVT::i32: return fastEmit_ARMISD_EH_SJLJ_SETJMP_MVT_i32_rr(RetVT, Op0, Op1);
3896 default: return Register();
3897 }
3898}
3899
3900// FastEmit functions for ARMISD::QADD16b.
3901
3902Register fastEmit_ARMISD_QADD16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3903 if (RetVT.SimpleTy != MVT::i32)
3904 return Register();
3905 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3906 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QADD16, RC: &ARM::rGPRRegClass, Op0, Op1);
3907 }
3908 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3909 return fastEmitInst_rr(MachineInstOpcode: ARM::QADD16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3910 }
3911 return Register();
3912}
3913
3914Register fastEmit_ARMISD_QADD16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3915 switch (VT.SimpleTy) {
3916 case MVT::i32: return fastEmit_ARMISD_QADD16b_MVT_i32_rr(RetVT, Op0, Op1);
3917 default: return Register();
3918 }
3919}
3920
3921// FastEmit functions for ARMISD::QADD8b.
3922
3923Register fastEmit_ARMISD_QADD8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3924 if (RetVT.SimpleTy != MVT::i32)
3925 return Register();
3926 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3927 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QADD8, RC: &ARM::rGPRRegClass, Op0, Op1);
3928 }
3929 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3930 return fastEmitInst_rr(MachineInstOpcode: ARM::QADD8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3931 }
3932 return Register();
3933}
3934
3935Register fastEmit_ARMISD_QADD8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3936 switch (VT.SimpleTy) {
3937 case MVT::i32: return fastEmit_ARMISD_QADD8b_MVT_i32_rr(RetVT, Op0, Op1);
3938 default: return Register();
3939 }
3940}
3941
3942// FastEmit functions for ARMISD::QSUB16b.
3943
3944Register fastEmit_ARMISD_QSUB16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3945 if (RetVT.SimpleTy != MVT::i32)
3946 return Register();
3947 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3948 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QSUB16, RC: &ARM::rGPRRegClass, Op0, Op1);
3949 }
3950 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3951 return fastEmitInst_rr(MachineInstOpcode: ARM::QSUB16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3952 }
3953 return Register();
3954}
3955
3956Register fastEmit_ARMISD_QSUB16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3957 switch (VT.SimpleTy) {
3958 case MVT::i32: return fastEmit_ARMISD_QSUB16b_MVT_i32_rr(RetVT, Op0, Op1);
3959 default: return Register();
3960 }
3961}
3962
3963// FastEmit functions for ARMISD::QSUB8b.
3964
3965Register fastEmit_ARMISD_QSUB8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3966 if (RetVT.SimpleTy != MVT::i32)
3967 return Register();
3968 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3969 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QSUB8, RC: &ARM::rGPRRegClass, Op0, Op1);
3970 }
3971 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3972 return fastEmitInst_rr(MachineInstOpcode: ARM::QSUB8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3973 }
3974 return Register();
3975}
3976
3977Register fastEmit_ARMISD_QSUB8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3978 switch (VT.SimpleTy) {
3979 case MVT::i32: return fastEmit_ARMISD_QSUB8b_MVT_i32_rr(RetVT, Op0, Op1);
3980 default: return Register();
3981 }
3982}
3983
3984// FastEmit functions for ARMISD::SMULWB.
3985
3986Register fastEmit_ARMISD_SMULWB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3987 if (RetVT.SimpleTy != MVT::i32)
3988 return Register();
3989 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3990 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SMULWB, RC: &ARM::rGPRRegClass, Op0, Op1);
3991 }
3992 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
3993 return fastEmitInst_rr(MachineInstOpcode: ARM::SMULWB, RC: &ARM::GPRRegClass, Op0, Op1);
3994 }
3995 return Register();
3996}
3997
3998Register fastEmit_ARMISD_SMULWB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3999 switch (VT.SimpleTy) {
4000 case MVT::i32: return fastEmit_ARMISD_SMULWB_MVT_i32_rr(RetVT, Op0, Op1);
4001 default: return Register();
4002 }
4003}
4004
4005// FastEmit functions for ARMISD::SMULWT.
4006
4007Register fastEmit_ARMISD_SMULWT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4008 if (RetVT.SimpleTy != MVT::i32)
4009 return Register();
4010 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4011 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SMULWT, RC: &ARM::rGPRRegClass, Op0, Op1);
4012 }
4013 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
4014 return fastEmitInst_rr(MachineInstOpcode: ARM::SMULWT, RC: &ARM::GPRRegClass, Op0, Op1);
4015 }
4016 return Register();
4017}
4018
4019Register fastEmit_ARMISD_SMULWT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4020 switch (VT.SimpleTy) {
4021 case MVT::i32: return fastEmit_ARMISD_SMULWT_MVT_i32_rr(RetVT, Op0, Op1);
4022 default: return Register();
4023 }
4024}
4025
4026// FastEmit functions for ARMISD::UQADD16b.
4027
4028Register fastEmit_ARMISD_UQADD16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4029 if (RetVT.SimpleTy != MVT::i32)
4030 return Register();
4031 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4032 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQADD16, RC: &ARM::rGPRRegClass, Op0, Op1);
4033 }
4034 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4035 return fastEmitInst_rr(MachineInstOpcode: ARM::UQADD16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4036 }
4037 return Register();
4038}
4039
4040Register fastEmit_ARMISD_UQADD16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4041 switch (VT.SimpleTy) {
4042 case MVT::i32: return fastEmit_ARMISD_UQADD16b_MVT_i32_rr(RetVT, Op0, Op1);
4043 default: return Register();
4044 }
4045}
4046
4047// FastEmit functions for ARMISD::UQADD8b.
4048
4049Register fastEmit_ARMISD_UQADD8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4050 if (RetVT.SimpleTy != MVT::i32)
4051 return Register();
4052 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4053 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQADD8, RC: &ARM::rGPRRegClass, Op0, Op1);
4054 }
4055 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4056 return fastEmitInst_rr(MachineInstOpcode: ARM::UQADD8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4057 }
4058 return Register();
4059}
4060
4061Register fastEmit_ARMISD_UQADD8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4062 switch (VT.SimpleTy) {
4063 case MVT::i32: return fastEmit_ARMISD_UQADD8b_MVT_i32_rr(RetVT, Op0, Op1);
4064 default: return Register();
4065 }
4066}
4067
4068// FastEmit functions for ARMISD::UQSUB16b.
4069
4070Register fastEmit_ARMISD_UQSUB16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4071 if (RetVT.SimpleTy != MVT::i32)
4072 return Register();
4073 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4074 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQSUB16, RC: &ARM::rGPRRegClass, Op0, Op1);
4075 }
4076 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4077 return fastEmitInst_rr(MachineInstOpcode: ARM::UQSUB16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4078 }
4079 return Register();
4080}
4081
4082Register fastEmit_ARMISD_UQSUB16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4083 switch (VT.SimpleTy) {
4084 case MVT::i32: return fastEmit_ARMISD_UQSUB16b_MVT_i32_rr(RetVT, Op0, Op1);
4085 default: return Register();
4086 }
4087}
4088
4089// FastEmit functions for ARMISD::UQSUB8b.
4090
4091Register fastEmit_ARMISD_UQSUB8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4092 if (RetVT.SimpleTy != MVT::i32)
4093 return Register();
4094 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4095 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQSUB8, RC: &ARM::rGPRRegClass, Op0, Op1);
4096 }
4097 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4098 return fastEmitInst_rr(MachineInstOpcode: ARM::UQSUB8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4099 }
4100 return Register();
4101}
4102
4103Register fastEmit_ARMISD_UQSUB8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4104 switch (VT.SimpleTy) {
4105 case MVT::i32: return fastEmit_ARMISD_UQSUB8b_MVT_i32_rr(RetVT, Op0, Op1);
4106 default: return Register();
4107 }
4108}
4109
4110// FastEmit functions for ARMISD::VMLAVs.
4111
4112Register fastEmit_ARMISD_VMLAVs_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4113 if (RetVT.SimpleTy != MVT::i32)
4114 return Register();
4115 if ((Subtarget->hasMVEIntegerOps())) {
4116 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVs8, RC: &ARM::tGPREvenRegClass, Op0, Op1);
4117 }
4118 return Register();
4119}
4120
4121Register fastEmit_ARMISD_VMLAVs_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4122 if (RetVT.SimpleTy != MVT::i32)
4123 return Register();
4124 if ((Subtarget->hasMVEIntegerOps())) {
4125 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVs16, RC: &ARM::tGPREvenRegClass, Op0, Op1);
4126 }
4127 return Register();
4128}
4129
4130Register fastEmit_ARMISD_VMLAVs_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4131 switch (VT.SimpleTy) {
4132 case MVT::v16i8: return fastEmit_ARMISD_VMLAVs_MVT_v16i8_rr(RetVT, Op0, Op1);
4133 case MVT::v8i16: return fastEmit_ARMISD_VMLAVs_MVT_v8i16_rr(RetVT, Op0, Op1);
4134 default: return Register();
4135 }
4136}
4137
4138// FastEmit functions for ARMISD::VMLAVu.
4139
4140Register fastEmit_ARMISD_VMLAVu_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4141 if (RetVT.SimpleTy != MVT::i32)
4142 return Register();
4143 if ((Subtarget->hasMVEIntegerOps())) {
4144 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVu8, RC: &ARM::tGPREvenRegClass, Op0, Op1);
4145 }
4146 return Register();
4147}
4148
4149Register fastEmit_ARMISD_VMLAVu_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4150 if (RetVT.SimpleTy != MVT::i32)
4151 return Register();
4152 if ((Subtarget->hasMVEIntegerOps())) {
4153 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVu16, RC: &ARM::tGPREvenRegClass, Op0, Op1);
4154 }
4155 return Register();
4156}
4157
4158Register fastEmit_ARMISD_VMLAVu_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4159 switch (VT.SimpleTy) {
4160 case MVT::v16i8: return fastEmit_ARMISD_VMLAVu_MVT_v16i8_rr(RetVT, Op0, Op1);
4161 case MVT::v8i16: return fastEmit_ARMISD_VMLAVu_MVT_v8i16_rr(RetVT, Op0, Op1);
4162 default: return Register();
4163 }
4164}
4165
4166// FastEmit functions for ARMISD::VMOVDRR.
4167
4168Register fastEmit_ARMISD_VMOVDRR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4169 if (RetVT.SimpleTy != MVT::f64)
4170 return Register();
4171 if ((Subtarget->hasFPRegs())) {
4172 return fastEmitInst_rr(MachineInstOpcode: ARM::VMOVDRR, RC: &ARM::DPRRegClass, Op0, Op1);
4173 }
4174 return Register();
4175}
4176
4177Register fastEmit_ARMISD_VMOVDRR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4178 switch (VT.SimpleTy) {
4179 case MVT::i32: return fastEmit_ARMISD_VMOVDRR_MVT_i32_rr(RetVT, Op0, Op1);
4180 default: return Register();
4181 }
4182}
4183
4184// FastEmit functions for ARMISD::VMULLs.
4185
4186Register fastEmit_ARMISD_VMULLs_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4187 if (RetVT.SimpleTy != MVT::v8i16)
4188 return Register();
4189 if ((Subtarget->hasNEON())) {
4190 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4191 }
4192 return Register();
4193}
4194
4195Register fastEmit_ARMISD_VMULLs_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4196 if (RetVT.SimpleTy != MVT::v4i32)
4197 return Register();
4198 if ((Subtarget->hasNEON())) {
4199 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4200 }
4201 return Register();
4202}
4203
4204Register fastEmit_ARMISD_VMULLs_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4205 if (RetVT.SimpleTy != MVT::v2i64)
4206 return Register();
4207 if ((Subtarget->hasNEON())) {
4208 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4209 }
4210 return Register();
4211}
4212
4213Register fastEmit_ARMISD_VMULLs_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4214 if (RetVT.SimpleTy != MVT::v2i64)
4215 return Register();
4216 if ((Subtarget->hasMVEIntegerOps())) {
4217 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULLBs32, RC: &ARM::MQPRRegClass, Op0, Op1);
4218 }
4219 return Register();
4220}
4221
4222Register fastEmit_ARMISD_VMULLs_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4223 switch (VT.SimpleTy) {
4224 case MVT::v8i8: return fastEmit_ARMISD_VMULLs_MVT_v8i8_rr(RetVT, Op0, Op1);
4225 case MVT::v4i16: return fastEmit_ARMISD_VMULLs_MVT_v4i16_rr(RetVT, Op0, Op1);
4226 case MVT::v2i32: return fastEmit_ARMISD_VMULLs_MVT_v2i32_rr(RetVT, Op0, Op1);
4227 case MVT::v4i32: return fastEmit_ARMISD_VMULLs_MVT_v4i32_rr(RetVT, Op0, Op1);
4228 default: return Register();
4229 }
4230}
4231
4232// FastEmit functions for ARMISD::VMULLu.
4233
4234Register fastEmit_ARMISD_VMULLu_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4235 if (RetVT.SimpleTy != MVT::v8i16)
4236 return Register();
4237 if ((Subtarget->hasNEON())) {
4238 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4239 }
4240 return Register();
4241}
4242
4243Register fastEmit_ARMISD_VMULLu_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4244 if (RetVT.SimpleTy != MVT::v4i32)
4245 return Register();
4246 if ((Subtarget->hasNEON())) {
4247 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4248 }
4249 return Register();
4250}
4251
4252Register fastEmit_ARMISD_VMULLu_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4253 if (RetVT.SimpleTy != MVT::v2i64)
4254 return Register();
4255 if ((Subtarget->hasNEON())) {
4256 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4257 }
4258 return Register();
4259}
4260
4261Register fastEmit_ARMISD_VMULLu_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4262 if (RetVT.SimpleTy != MVT::v2i64)
4263 return Register();
4264 if ((Subtarget->hasMVEIntegerOps())) {
4265 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULLBu32, RC: &ARM::MQPRRegClass, Op0, Op1);
4266 }
4267 return Register();
4268}
4269
4270Register fastEmit_ARMISD_VMULLu_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4271 switch (VT.SimpleTy) {
4272 case MVT::v8i8: return fastEmit_ARMISD_VMULLu_MVT_v8i8_rr(RetVT, Op0, Op1);
4273 case MVT::v4i16: return fastEmit_ARMISD_VMULLu_MVT_v4i16_rr(RetVT, Op0, Op1);
4274 case MVT::v2i32: return fastEmit_ARMISD_VMULLu_MVT_v2i32_rr(RetVT, Op0, Op1);
4275 case MVT::v4i32: return fastEmit_ARMISD_VMULLu_MVT_v4i32_rr(RetVT, Op0, Op1);
4276 default: return Register();
4277 }
4278}
4279
4280// FastEmit functions for ARMISD::VQDMULH.
4281
4282Register fastEmit_ARMISD_VQDMULH_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4283 if (RetVT.SimpleTy != MVT::v16i8)
4284 return Register();
4285 if ((Subtarget->hasMVEIntegerOps())) {
4286 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQDMULHi8, RC: &ARM::MQPRRegClass, Op0, Op1);
4287 }
4288 return Register();
4289}
4290
4291Register fastEmit_ARMISD_VQDMULH_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4292 if (RetVT.SimpleTy != MVT::v8i16)
4293 return Register();
4294 if ((Subtarget->hasMVEIntegerOps())) {
4295 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQDMULHi16, RC: &ARM::MQPRRegClass, Op0, Op1);
4296 }
4297 return Register();
4298}
4299
4300Register fastEmit_ARMISD_VQDMULH_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4301 if (RetVT.SimpleTy != MVT::v4i32)
4302 return Register();
4303 if ((Subtarget->hasMVEIntegerOps())) {
4304 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQDMULHi32, RC: &ARM::MQPRRegClass, Op0, Op1);
4305 }
4306 return Register();
4307}
4308
4309Register fastEmit_ARMISD_VQDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4310 switch (VT.SimpleTy) {
4311 case MVT::v16i8: return fastEmit_ARMISD_VQDMULH_MVT_v16i8_rr(RetVT, Op0, Op1);
4312 case MVT::v8i16: return fastEmit_ARMISD_VQDMULH_MVT_v8i16_rr(RetVT, Op0, Op1);
4313 case MVT::v4i32: return fastEmit_ARMISD_VQDMULH_MVT_v4i32_rr(RetVT, Op0, Op1);
4314 default: return Register();
4315 }
4316}
4317
4318// FastEmit functions for ARMISD::VSHLs.
4319
4320Register fastEmit_ARMISD_VSHLs_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4321 if (RetVT.SimpleTy != MVT::v8i8)
4322 return Register();
4323 if ((Subtarget->hasNEON())) {
4324 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4325 }
4326 return Register();
4327}
4328
4329Register fastEmit_ARMISD_VSHLs_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4330 if (RetVT.SimpleTy != MVT::v16i8)
4331 return Register();
4332 if ((Subtarget->hasMVEIntegerOps())) {
4333 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecs8, RC: &ARM::MQPRRegClass, Op0, Op1);
4334 }
4335 if ((Subtarget->hasNEON())) {
4336 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4337 }
4338 return Register();
4339}
4340
4341Register fastEmit_ARMISD_VSHLs_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4342 if (RetVT.SimpleTy != MVT::v4i16)
4343 return Register();
4344 if ((Subtarget->hasNEON())) {
4345 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4346 }
4347 return Register();
4348}
4349
4350Register fastEmit_ARMISD_VSHLs_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4351 if (RetVT.SimpleTy != MVT::v8i16)
4352 return Register();
4353 if ((Subtarget->hasMVEIntegerOps())) {
4354 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecs16, RC: &ARM::MQPRRegClass, Op0, Op1);
4355 }
4356 if ((Subtarget->hasNEON())) {
4357 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4358 }
4359 return Register();
4360}
4361
4362Register fastEmit_ARMISD_VSHLs_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4363 if (RetVT.SimpleTy != MVT::v2i32)
4364 return Register();
4365 if ((Subtarget->hasNEON())) {
4366 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4367 }
4368 return Register();
4369}
4370
4371Register fastEmit_ARMISD_VSHLs_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4372 if (RetVT.SimpleTy != MVT::v4i32)
4373 return Register();
4374 if ((Subtarget->hasMVEIntegerOps())) {
4375 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecs32, RC: &ARM::MQPRRegClass, Op0, Op1);
4376 }
4377 if ((Subtarget->hasNEON())) {
4378 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4379 }
4380 return Register();
4381}
4382
4383Register fastEmit_ARMISD_VSHLs_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
4384 if (RetVT.SimpleTy != MVT::v1i64)
4385 return Register();
4386 if ((Subtarget->hasNEON())) {
4387 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
4388 }
4389 return Register();
4390}
4391
4392Register fastEmit_ARMISD_VSHLs_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
4393 if (RetVT.SimpleTy != MVT::v2i64)
4394 return Register();
4395 if ((Subtarget->hasNEON())) {
4396 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4397 }
4398 return Register();
4399}
4400
4401Register fastEmit_ARMISD_VSHLs_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4402 switch (VT.SimpleTy) {
4403 case MVT::v8i8: return fastEmit_ARMISD_VSHLs_MVT_v8i8_rr(RetVT, Op0, Op1);
4404 case MVT::v16i8: return fastEmit_ARMISD_VSHLs_MVT_v16i8_rr(RetVT, Op0, Op1);
4405 case MVT::v4i16: return fastEmit_ARMISD_VSHLs_MVT_v4i16_rr(RetVT, Op0, Op1);
4406 case MVT::v8i16: return fastEmit_ARMISD_VSHLs_MVT_v8i16_rr(RetVT, Op0, Op1);
4407 case MVT::v2i32: return fastEmit_ARMISD_VSHLs_MVT_v2i32_rr(RetVT, Op0, Op1);
4408 case MVT::v4i32: return fastEmit_ARMISD_VSHLs_MVT_v4i32_rr(RetVT, Op0, Op1);
4409 case MVT::v1i64: return fastEmit_ARMISD_VSHLs_MVT_v1i64_rr(RetVT, Op0, Op1);
4410 case MVT::v2i64: return fastEmit_ARMISD_VSHLs_MVT_v2i64_rr(RetVT, Op0, Op1);
4411 default: return Register();
4412 }
4413}
4414
4415// FastEmit functions for ARMISD::VSHLu.
4416
4417Register fastEmit_ARMISD_VSHLu_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4418 if (RetVT.SimpleTy != MVT::v8i8)
4419 return Register();
4420 if ((Subtarget->hasNEON())) {
4421 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4422 }
4423 return Register();
4424}
4425
4426Register fastEmit_ARMISD_VSHLu_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4427 if (RetVT.SimpleTy != MVT::v16i8)
4428 return Register();
4429 if ((Subtarget->hasMVEIntegerOps())) {
4430 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecu8, RC: &ARM::MQPRRegClass, Op0, Op1);
4431 }
4432 if ((Subtarget->hasNEON())) {
4433 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4434 }
4435 return Register();
4436}
4437
4438Register fastEmit_ARMISD_VSHLu_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4439 if (RetVT.SimpleTy != MVT::v4i16)
4440 return Register();
4441 if ((Subtarget->hasNEON())) {
4442 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4443 }
4444 return Register();
4445}
4446
4447Register fastEmit_ARMISD_VSHLu_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4448 if (RetVT.SimpleTy != MVT::v8i16)
4449 return Register();
4450 if ((Subtarget->hasMVEIntegerOps())) {
4451 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecu16, RC: &ARM::MQPRRegClass, Op0, Op1);
4452 }
4453 if ((Subtarget->hasNEON())) {
4454 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4455 }
4456 return Register();
4457}
4458
4459Register fastEmit_ARMISD_VSHLu_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4460 if (RetVT.SimpleTy != MVT::v2i32)
4461 return Register();
4462 if ((Subtarget->hasNEON())) {
4463 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4464 }
4465 return Register();
4466}
4467
4468Register fastEmit_ARMISD_VSHLu_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4469 if (RetVT.SimpleTy != MVT::v4i32)
4470 return Register();
4471 if ((Subtarget->hasMVEIntegerOps())) {
4472 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecu32, RC: &ARM::MQPRRegClass, Op0, Op1);
4473 }
4474 if ((Subtarget->hasNEON())) {
4475 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4476 }
4477 return Register();
4478}
4479
4480Register fastEmit_ARMISD_VSHLu_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
4481 if (RetVT.SimpleTy != MVT::v1i64)
4482 return Register();
4483 if ((Subtarget->hasNEON())) {
4484 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
4485 }
4486 return Register();
4487}
4488
4489Register fastEmit_ARMISD_VSHLu_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
4490 if (RetVT.SimpleTy != MVT::v2i64)
4491 return Register();
4492 if ((Subtarget->hasNEON())) {
4493 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4494 }
4495 return Register();
4496}
4497
4498Register fastEmit_ARMISD_VSHLu_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4499 switch (VT.SimpleTy) {
4500 case MVT::v8i8: return fastEmit_ARMISD_VSHLu_MVT_v8i8_rr(RetVT, Op0, Op1);
4501 case MVT::v16i8: return fastEmit_ARMISD_VSHLu_MVT_v16i8_rr(RetVT, Op0, Op1);
4502 case MVT::v4i16: return fastEmit_ARMISD_VSHLu_MVT_v4i16_rr(RetVT, Op0, Op1);
4503 case MVT::v8i16: return fastEmit_ARMISD_VSHLu_MVT_v8i16_rr(RetVT, Op0, Op1);
4504 case MVT::v2i32: return fastEmit_ARMISD_VSHLu_MVT_v2i32_rr(RetVT, Op0, Op1);
4505 case MVT::v4i32: return fastEmit_ARMISD_VSHLu_MVT_v4i32_rr(RetVT, Op0, Op1);
4506 case MVT::v1i64: return fastEmit_ARMISD_VSHLu_MVT_v1i64_rr(RetVT, Op0, Op1);
4507 case MVT::v2i64: return fastEmit_ARMISD_VSHLu_MVT_v2i64_rr(RetVT, Op0, Op1);
4508 default: return Register();
4509 }
4510}
4511
4512// FastEmit functions for ARMISD::VTBL1.
4513
4514Register fastEmit_ARMISD_VTBL1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4515 if (RetVT.SimpleTy != MVT::v8i8)
4516 return Register();
4517 if ((Subtarget->hasNEON())) {
4518 return fastEmitInst_rr(MachineInstOpcode: ARM::VTBL1, RC: &ARM::DPRRegClass, Op0, Op1);
4519 }
4520 return Register();
4521}
4522
4523Register fastEmit_ARMISD_VTBL1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4524 switch (VT.SimpleTy) {
4525 case MVT::v8i8: return fastEmit_ARMISD_VTBL1_MVT_v8i8_rr(RetVT, Op0, Op1);
4526 default: return Register();
4527 }
4528}
4529
4530// FastEmit functions for ARMISD::VTST.
4531
4532Register fastEmit_ARMISD_VTST_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4533 if (RetVT.SimpleTy != MVT::v8i8)
4534 return Register();
4535 if ((Subtarget->hasNEON())) {
4536 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4537 }
4538 return Register();
4539}
4540
4541Register fastEmit_ARMISD_VTST_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4542 if (RetVT.SimpleTy != MVT::v16i8)
4543 return Register();
4544 if ((Subtarget->hasNEON())) {
4545 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4546 }
4547 return Register();
4548}
4549
4550Register fastEmit_ARMISD_VTST_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4551 if (RetVT.SimpleTy != MVT::v4i16)
4552 return Register();
4553 if ((Subtarget->hasNEON())) {
4554 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4555 }
4556 return Register();
4557}
4558
4559Register fastEmit_ARMISD_VTST_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4560 if (RetVT.SimpleTy != MVT::v8i16)
4561 return Register();
4562 if ((Subtarget->hasNEON())) {
4563 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4564 }
4565 return Register();
4566}
4567
4568Register fastEmit_ARMISD_VTST_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4569 if (RetVT.SimpleTy != MVT::v2i32)
4570 return Register();
4571 if ((Subtarget->hasNEON())) {
4572 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4573 }
4574 return Register();
4575}
4576
4577Register fastEmit_ARMISD_VTST_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4578 if (RetVT.SimpleTy != MVT::v4i32)
4579 return Register();
4580 if ((Subtarget->hasNEON())) {
4581 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4582 }
4583 return Register();
4584}
4585
4586Register fastEmit_ARMISD_VTST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4587 switch (VT.SimpleTy) {
4588 case MVT::v8i8: return fastEmit_ARMISD_VTST_MVT_v8i8_rr(RetVT, Op0, Op1);
4589 case MVT::v16i8: return fastEmit_ARMISD_VTST_MVT_v16i8_rr(RetVT, Op0, Op1);
4590 case MVT::v4i16: return fastEmit_ARMISD_VTST_MVT_v4i16_rr(RetVT, Op0, Op1);
4591 case MVT::v8i16: return fastEmit_ARMISD_VTST_MVT_v8i16_rr(RetVT, Op0, Op1);
4592 case MVT::v2i32: return fastEmit_ARMISD_VTST_MVT_v2i32_rr(RetVT, Op0, Op1);
4593 case MVT::v4i32: return fastEmit_ARMISD_VTST_MVT_v4i32_rr(RetVT, Op0, Op1);
4594 default: return Register();
4595 }
4596}
4597
4598// FastEmit functions for ISD::ABDS.
4599
4600Register fastEmit_ISD_ABDS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4601 if (RetVT.SimpleTy != MVT::v8i8)
4602 return Register();
4603 if ((Subtarget->hasNEON())) {
4604 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4605 }
4606 return Register();
4607}
4608
4609Register fastEmit_ISD_ABDS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4610 if (RetVT.SimpleTy != MVT::v16i8)
4611 return Register();
4612 if ((Subtarget->hasMVEIntegerOps())) {
4613 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
4614 }
4615 if ((Subtarget->hasNEON())) {
4616 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4617 }
4618 return Register();
4619}
4620
4621Register fastEmit_ISD_ABDS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4622 if (RetVT.SimpleTy != MVT::v4i16)
4623 return Register();
4624 if ((Subtarget->hasNEON())) {
4625 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4626 }
4627 return Register();
4628}
4629
4630Register fastEmit_ISD_ABDS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4631 if (RetVT.SimpleTy != MVT::v8i16)
4632 return Register();
4633 if ((Subtarget->hasMVEIntegerOps())) {
4634 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
4635 }
4636 if ((Subtarget->hasNEON())) {
4637 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4638 }
4639 return Register();
4640}
4641
4642Register fastEmit_ISD_ABDS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4643 if (RetVT.SimpleTy != MVT::v2i32)
4644 return Register();
4645 if ((Subtarget->hasNEON())) {
4646 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4647 }
4648 return Register();
4649}
4650
4651Register fastEmit_ISD_ABDS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4652 if (RetVT.SimpleTy != MVT::v4i32)
4653 return Register();
4654 if ((Subtarget->hasMVEIntegerOps())) {
4655 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
4656 }
4657 if ((Subtarget->hasNEON())) {
4658 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4659 }
4660 return Register();
4661}
4662
4663Register fastEmit_ISD_ABDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4664 switch (VT.SimpleTy) {
4665 case MVT::v8i8: return fastEmit_ISD_ABDS_MVT_v8i8_rr(RetVT, Op0, Op1);
4666 case MVT::v16i8: return fastEmit_ISD_ABDS_MVT_v16i8_rr(RetVT, Op0, Op1);
4667 case MVT::v4i16: return fastEmit_ISD_ABDS_MVT_v4i16_rr(RetVT, Op0, Op1);
4668 case MVT::v8i16: return fastEmit_ISD_ABDS_MVT_v8i16_rr(RetVT, Op0, Op1);
4669 case MVT::v2i32: return fastEmit_ISD_ABDS_MVT_v2i32_rr(RetVT, Op0, Op1);
4670 case MVT::v4i32: return fastEmit_ISD_ABDS_MVT_v4i32_rr(RetVT, Op0, Op1);
4671 default: return Register();
4672 }
4673}
4674
4675// FastEmit functions for ISD::ABDU.
4676
4677Register fastEmit_ISD_ABDU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4678 if (RetVT.SimpleTy != MVT::v8i8)
4679 return Register();
4680 if ((Subtarget->hasNEON())) {
4681 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4682 }
4683 return Register();
4684}
4685
4686Register fastEmit_ISD_ABDU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4687 if (RetVT.SimpleTy != MVT::v16i8)
4688 return Register();
4689 if ((Subtarget->hasMVEIntegerOps())) {
4690 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
4691 }
4692 if ((Subtarget->hasNEON())) {
4693 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4694 }
4695 return Register();
4696}
4697
4698Register fastEmit_ISD_ABDU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4699 if (RetVT.SimpleTy != MVT::v4i16)
4700 return Register();
4701 if ((Subtarget->hasNEON())) {
4702 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4703 }
4704 return Register();
4705}
4706
4707Register fastEmit_ISD_ABDU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4708 if (RetVT.SimpleTy != MVT::v8i16)
4709 return Register();
4710 if ((Subtarget->hasMVEIntegerOps())) {
4711 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
4712 }
4713 if ((Subtarget->hasNEON())) {
4714 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4715 }
4716 return Register();
4717}
4718
4719Register fastEmit_ISD_ABDU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4720 if (RetVT.SimpleTy != MVT::v2i32)
4721 return Register();
4722 if ((Subtarget->hasNEON())) {
4723 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4724 }
4725 return Register();
4726}
4727
4728Register fastEmit_ISD_ABDU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4729 if (RetVT.SimpleTy != MVT::v4i32)
4730 return Register();
4731 if ((Subtarget->hasMVEIntegerOps())) {
4732 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
4733 }
4734 if ((Subtarget->hasNEON())) {
4735 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4736 }
4737 return Register();
4738}
4739
4740Register fastEmit_ISD_ABDU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4741 switch (VT.SimpleTy) {
4742 case MVT::v8i8: return fastEmit_ISD_ABDU_MVT_v8i8_rr(RetVT, Op0, Op1);
4743 case MVT::v16i8: return fastEmit_ISD_ABDU_MVT_v16i8_rr(RetVT, Op0, Op1);
4744 case MVT::v4i16: return fastEmit_ISD_ABDU_MVT_v4i16_rr(RetVT, Op0, Op1);
4745 case MVT::v8i16: return fastEmit_ISD_ABDU_MVT_v8i16_rr(RetVT, Op0, Op1);
4746 case MVT::v2i32: return fastEmit_ISD_ABDU_MVT_v2i32_rr(RetVT, Op0, Op1);
4747 case MVT::v4i32: return fastEmit_ISD_ABDU_MVT_v4i32_rr(RetVT, Op0, Op1);
4748 default: return Register();
4749 }
4750}
4751
4752// FastEmit functions for ISD::ADD.
4753
4754Register fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4755 if (RetVT.SimpleTy != MVT::i32)
4756 return Register();
4757 if ((Subtarget->isThumb2())) {
4758 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ADDrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4759 }
4760 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
4761 return fastEmitInst_rr(MachineInstOpcode: ARM::tADDrr, RC: &ARM::tGPRRegClass, Op0, Op1);
4762 }
4763 if ((!Subtarget->isThumb())) {
4764 return fastEmitInst_rr(MachineInstOpcode: ARM::ADDrr, RC: &ARM::GPRRegClass, Op0, Op1);
4765 }
4766 return Register();
4767}
4768
4769Register fastEmit_ISD_ADD_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4770 if (RetVT.SimpleTy != MVT::v8i8)
4771 return Register();
4772 if ((Subtarget->hasNEON())) {
4773 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4774 }
4775 return Register();
4776}
4777
4778Register fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4779 if (RetVT.SimpleTy != MVT::v16i8)
4780 return Register();
4781 if ((Subtarget->hasMVEIntegerOps())) {
4782 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDi8, RC: &ARM::MQPRRegClass, Op0, Op1);
4783 }
4784 if ((Subtarget->hasNEON())) {
4785 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4786 }
4787 return Register();
4788}
4789
4790Register fastEmit_ISD_ADD_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4791 if (RetVT.SimpleTy != MVT::v4i16)
4792 return Register();
4793 if ((Subtarget->hasNEON())) {
4794 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4795 }
4796 return Register();
4797}
4798
4799Register fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4800 if (RetVT.SimpleTy != MVT::v8i16)
4801 return Register();
4802 if ((Subtarget->hasMVEIntegerOps())) {
4803 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDi16, RC: &ARM::MQPRRegClass, Op0, Op1);
4804 }
4805 if ((Subtarget->hasNEON())) {
4806 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4807 }
4808 return Register();
4809}
4810
4811Register fastEmit_ISD_ADD_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4812 if (RetVT.SimpleTy != MVT::v2i32)
4813 return Register();
4814 if ((Subtarget->hasNEON())) {
4815 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4816 }
4817 return Register();
4818}
4819
4820Register fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4821 if (RetVT.SimpleTy != MVT::v4i32)
4822 return Register();
4823 if ((Subtarget->hasMVEIntegerOps())) {
4824 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDi32, RC: &ARM::MQPRRegClass, Op0, Op1);
4825 }
4826 if ((Subtarget->hasNEON())) {
4827 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4828 }
4829 return Register();
4830}
4831
4832Register fastEmit_ISD_ADD_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
4833 if (RetVT.SimpleTy != MVT::v1i64)
4834 return Register();
4835 if ((Subtarget->hasNEON())) {
4836 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
4837 }
4838 return Register();
4839}
4840
4841Register fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
4842 if (RetVT.SimpleTy != MVT::v2i64)
4843 return Register();
4844 if ((Subtarget->hasNEON())) {
4845 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4846 }
4847 return Register();
4848}
4849
4850Register fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4851 switch (VT.SimpleTy) {
4852 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
4853 case MVT::v8i8: return fastEmit_ISD_ADD_MVT_v8i8_rr(RetVT, Op0, Op1);
4854 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
4855 case MVT::v4i16: return fastEmit_ISD_ADD_MVT_v4i16_rr(RetVT, Op0, Op1);
4856 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
4857 case MVT::v2i32: return fastEmit_ISD_ADD_MVT_v2i32_rr(RetVT, Op0, Op1);
4858 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
4859 case MVT::v1i64: return fastEmit_ISD_ADD_MVT_v1i64_rr(RetVT, Op0, Op1);
4860 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
4861 default: return Register();
4862 }
4863}
4864
4865// FastEmit functions for ISD::AND.
4866
4867Register fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4868 if (RetVT.SimpleTy != MVT::i32)
4869 return Register();
4870 if ((Subtarget->isThumb2())) {
4871 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ANDrr, RC: &ARM::rGPRRegClass, Op0, Op1);
4872 }
4873 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
4874 return fastEmitInst_rr(MachineInstOpcode: ARM::tAND, RC: &ARM::tGPRRegClass, Op0, Op1);
4875 }
4876 if ((!Subtarget->isThumb())) {
4877 return fastEmitInst_rr(MachineInstOpcode: ARM::ANDrr, RC: &ARM::GPRRegClass, Op0, Op1);
4878 }
4879 return Register();
4880}
4881
4882Register fastEmit_ISD_AND_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4883 if (RetVT.SimpleTy != MVT::v8i8)
4884 return Register();
4885 if ((Subtarget->hasNEON())) {
4886 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
4887 }
4888 return Register();
4889}
4890
4891Register fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4892 if (RetVT.SimpleTy != MVT::v16i8)
4893 return Register();
4894 if ((Subtarget->hasMVEIntegerOps())) {
4895 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
4896 }
4897 if ((Subtarget->hasNEON())) {
4898 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
4899 }
4900 return Register();
4901}
4902
4903Register fastEmit_ISD_AND_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4904 if (RetVT.SimpleTy != MVT::v4i16)
4905 return Register();
4906 if ((Subtarget->hasNEON())) {
4907 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
4908 }
4909 return Register();
4910}
4911
4912Register fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4913 if (RetVT.SimpleTy != MVT::v8i16)
4914 return Register();
4915 if ((Subtarget->hasMVEIntegerOps())) {
4916 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
4917 }
4918 if ((Subtarget->hasNEON())) {
4919 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
4920 }
4921 return Register();
4922}
4923
4924Register fastEmit_ISD_AND_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4925 if (RetVT.SimpleTy != MVT::v2i32)
4926 return Register();
4927 if ((Subtarget->hasNEON())) {
4928 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
4929 }
4930 return Register();
4931}
4932
4933Register fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4934 if (RetVT.SimpleTy != MVT::v4i32)
4935 return Register();
4936 if ((Subtarget->hasMVEIntegerOps())) {
4937 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
4938 }
4939 if ((Subtarget->hasNEON())) {
4940 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
4941 }
4942 return Register();
4943}
4944
4945Register fastEmit_ISD_AND_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
4946 if (RetVT.SimpleTy != MVT::v1i64)
4947 return Register();
4948 if ((Subtarget->hasNEON())) {
4949 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
4950 }
4951 return Register();
4952}
4953
4954Register fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
4955 if (RetVT.SimpleTy != MVT::v2i64)
4956 return Register();
4957 if ((Subtarget->hasMVEIntegerOps())) {
4958 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
4959 }
4960 if ((Subtarget->hasNEON())) {
4961 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
4962 }
4963 return Register();
4964}
4965
4966Register fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4967 switch (VT.SimpleTy) {
4968 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
4969 case MVT::v8i8: return fastEmit_ISD_AND_MVT_v8i8_rr(RetVT, Op0, Op1);
4970 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
4971 case MVT::v4i16: return fastEmit_ISD_AND_MVT_v4i16_rr(RetVT, Op0, Op1);
4972 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
4973 case MVT::v2i32: return fastEmit_ISD_AND_MVT_v2i32_rr(RetVT, Op0, Op1);
4974 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
4975 case MVT::v1i64: return fastEmit_ISD_AND_MVT_v1i64_rr(RetVT, Op0, Op1);
4976 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
4977 default: return Register();
4978 }
4979}
4980
4981// FastEmit functions for ISD::AVGCEILS.
4982
4983Register fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4984 if (RetVT.SimpleTy != MVT::v16i8)
4985 return Register();
4986 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
4987}
4988
4989Register fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4990 if (RetVT.SimpleTy != MVT::v8i16)
4991 return Register();
4992 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
4993}
4994
4995Register fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4996 if (RetVT.SimpleTy != MVT::v4i32)
4997 return Register();
4998 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
4999}
5000
5001Register fastEmit_ISD_AVGCEILS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5002 switch (VT.SimpleTy) {
5003 case MVT::v16i8: return fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(RetVT, Op0, Op1);
5004 case MVT::v8i16: return fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(RetVT, Op0, Op1);
5005 case MVT::v4i32: return fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(RetVT, Op0, Op1);
5006 default: return Register();
5007 }
5008}
5009
5010// FastEmit functions for ISD::AVGCEILU.
5011
5012Register fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5013 if (RetVT.SimpleTy != MVT::v16i8)
5014 return Register();
5015 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
5016}
5017
5018Register fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5019 if (RetVT.SimpleTy != MVT::v8i16)
5020 return Register();
5021 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
5022}
5023
5024Register fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5025 if (RetVT.SimpleTy != MVT::v4i32)
5026 return Register();
5027 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
5028}
5029
5030Register fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5031 switch (VT.SimpleTy) {
5032 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
5033 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
5034 case MVT::v4i32: return fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(RetVT, Op0, Op1);
5035 default: return Register();
5036 }
5037}
5038
5039// FastEmit functions for ISD::AVGFLOORS.
5040
5041Register fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5042 if (RetVT.SimpleTy != MVT::v16i8)
5043 return Register();
5044 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
5045}
5046
5047Register fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5048 if (RetVT.SimpleTy != MVT::v8i16)
5049 return Register();
5050 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
5051}
5052
5053Register fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5054 if (RetVT.SimpleTy != MVT::v4i32)
5055 return Register();
5056 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
5057}
5058
5059Register fastEmit_ISD_AVGFLOORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5060 switch (VT.SimpleTy) {
5061 case MVT::v16i8: return fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(RetVT, Op0, Op1);
5062 case MVT::v8i16: return fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(RetVT, Op0, Op1);
5063 case MVT::v4i32: return fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(RetVT, Op0, Op1);
5064 default: return Register();
5065 }
5066}
5067
5068// FastEmit functions for ISD::AVGFLOORU.
5069
5070Register fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5071 if (RetVT.SimpleTy != MVT::v16i8)
5072 return Register();
5073 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
5074}
5075
5076Register fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5077 if (RetVT.SimpleTy != MVT::v8i16)
5078 return Register();
5079 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
5080}
5081
5082Register fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5083 if (RetVT.SimpleTy != MVT::v4i32)
5084 return Register();
5085 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
5086}
5087
5088Register fastEmit_ISD_AVGFLOORU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5089 switch (VT.SimpleTy) {
5090 case MVT::v16i8: return fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(RetVT, Op0, Op1);
5091 case MVT::v8i16: return fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(RetVT, Op0, Op1);
5092 case MVT::v4i32: return fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(RetVT, Op0, Op1);
5093 default: return Register();
5094 }
5095}
5096
5097// FastEmit functions for ISD::FADD.
5098
5099Register fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5100 if (RetVT.SimpleTy != MVT::f16)
5101 return Register();
5102 if ((Subtarget->hasFullFP16())) {
5103 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDH, RC: &ARM::HPRRegClass, Op0, Op1);
5104 }
5105 return Register();
5106}
5107
5108Register fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5109 if (RetVT.SimpleTy != MVT::f32)
5110 return Register();
5111 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
5112 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDS, RC: &ARM::SPRRegClass, Op0, Op1);
5113 }
5114 return Register();
5115}
5116
5117Register fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5118 if (RetVT.SimpleTy != MVT::f64)
5119 return Register();
5120 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
5121 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDD, RC: &ARM::DPRRegClass, Op0, Op1);
5122 }
5123 return Register();
5124}
5125
5126Register fastEmit_ISD_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5127 if (RetVT.SimpleTy != MVT::v4f16)
5128 return Register();
5129 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5130 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDhd, RC: &ARM::DPRRegClass, Op0, Op1);
5131 }
5132 return Register();
5133}
5134
5135Register fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5136 if (RetVT.SimpleTy != MVT::v8f16)
5137 return Register();
5138 if ((Subtarget->hasMVEFloatOps())) {
5139 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5140 }
5141 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5142 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDhq, RC: &ARM::QPRRegClass, Op0, Op1);
5143 }
5144 return Register();
5145}
5146
5147Register fastEmit_ISD_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5148 if (RetVT.SimpleTy != MVT::v2f32)
5149 return Register();
5150 if ((Subtarget->hasNEON())) {
5151 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDfd, RC: &ARM::DPRRegClass, Op0, Op1);
5152 }
5153 return Register();
5154}
5155
5156Register fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5157 if (RetVT.SimpleTy != MVT::v4f32)
5158 return Register();
5159 if ((Subtarget->hasMVEFloatOps())) {
5160 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5161 }
5162 if ((Subtarget->hasNEON())) {
5163 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDfq, RC: &ARM::QPRRegClass, Op0, Op1);
5164 }
5165 return Register();
5166}
5167
5168Register fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5169 switch (VT.SimpleTy) {
5170 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
5171 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
5172 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
5173 case MVT::v4f16: return fastEmit_ISD_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
5174 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
5175 case MVT::v2f32: return fastEmit_ISD_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
5176 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
5177 default: return Register();
5178 }
5179}
5180
5181// FastEmit functions for ISD::FDIV.
5182
5183Register fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5184 if (RetVT.SimpleTy != MVT::f16)
5185 return Register();
5186 if ((Subtarget->hasFullFP16())) {
5187 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVH, RC: &ARM::HPRRegClass, Op0, Op1);
5188 }
5189 return Register();
5190}
5191
5192Register fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5193 if (RetVT.SimpleTy != MVT::f32)
5194 return Register();
5195 if ((Subtarget->hasVFP2Base())) {
5196 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVS, RC: &ARM::SPRRegClass, Op0, Op1);
5197 }
5198 return Register();
5199}
5200
5201Register fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5202 if (RetVT.SimpleTy != MVT::f64)
5203 return Register();
5204 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
5205 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVD, RC: &ARM::DPRRegClass, Op0, Op1);
5206 }
5207 return Register();
5208}
5209
5210Register fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5211 switch (VT.SimpleTy) {
5212 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
5213 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
5214 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
5215 default: return Register();
5216 }
5217}
5218
5219// FastEmit functions for ISD::FMAXIMUM.
5220
5221Register fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5222 if (RetVT.SimpleTy != MVT::v4f16)
5223 return Register();
5224 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5225 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXhd, RC: &ARM::DPRRegClass, Op0, Op1);
5226 }
5227 return Register();
5228}
5229
5230Register fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5231 if (RetVT.SimpleTy != MVT::v8f16)
5232 return Register();
5233 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5234 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXhq, RC: &ARM::QPRRegClass, Op0, Op1);
5235 }
5236 return Register();
5237}
5238
5239Register fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5240 if (RetVT.SimpleTy != MVT::v2f32)
5241 return Register();
5242 if ((Subtarget->hasNEON())) {
5243 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXfd, RC: &ARM::DPRRegClass, Op0, Op1);
5244 }
5245 return Register();
5246}
5247
5248Register fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5249 if (RetVT.SimpleTy != MVT::v4f32)
5250 return Register();
5251 if ((Subtarget->hasNEON())) {
5252 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXfq, RC: &ARM::QPRRegClass, Op0, Op1);
5253 }
5254 return Register();
5255}
5256
5257Register fastEmit_ISD_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5258 switch (VT.SimpleTy) {
5259 case MVT::v4f16: return fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
5260 case MVT::v8f16: return fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
5261 case MVT::v2f32: return fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
5262 case MVT::v4f32: return fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
5263 default: return Register();
5264 }
5265}
5266
5267// FastEmit functions for ISD::FMAXNUM.
5268
5269Register fastEmit_ISD_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5270 if (RetVT.SimpleTy != MVT::f16)
5271 return Register();
5272 if ((Subtarget->hasFullFP16())) {
5273 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMH, RC: &ARM::HPRRegClass, Op0, Op1);
5274 }
5275 return Register();
5276}
5277
5278Register fastEmit_ISD_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5279 if (RetVT.SimpleTy != MVT::f32)
5280 return Register();
5281 if ((Subtarget->hasFPARMv8Base())) {
5282 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMS, RC: &ARM::SPRRegClass, Op0, Op1);
5283 }
5284 return Register();
5285}
5286
5287Register fastEmit_ISD_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5288 if (RetVT.SimpleTy != MVT::f64)
5289 return Register();
5290 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
5291 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMD, RC: &ARM::DPRRegClass, Op0, Op1);
5292 }
5293 return Register();
5294}
5295
5296Register fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5297 if (RetVT.SimpleTy != MVT::v4f16)
5298 return Register();
5299 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5300 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNDh, RC: &ARM::DPRRegClass, Op0, Op1);
5301 }
5302 return Register();
5303}
5304
5305Register fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5306 if (RetVT.SimpleTy != MVT::v8f16)
5307 return Register();
5308 if ((Subtarget->hasMVEFloatOps())) {
5309 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXNMf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5310 }
5311 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5312 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNQh, RC: &ARM::QPRRegClass, Op0, Op1);
5313 }
5314 return Register();
5315}
5316
5317Register fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5318 if (RetVT.SimpleTy != MVT::v2f32)
5319 return Register();
5320 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
5321 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNDf, RC: &ARM::DPRRegClass, Op0, Op1);
5322 }
5323 return Register();
5324}
5325
5326Register fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5327 if (RetVT.SimpleTy != MVT::v4f32)
5328 return Register();
5329 if ((Subtarget->hasMVEFloatOps())) {
5330 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXNMf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5331 }
5332 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
5333 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNQf, RC: &ARM::QPRRegClass, Op0, Op1);
5334 }
5335 return Register();
5336}
5337
5338Register fastEmit_ISD_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5339 switch (VT.SimpleTy) {
5340 case MVT::f16: return fastEmit_ISD_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
5341 case MVT::f32: return fastEmit_ISD_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
5342 case MVT::f64: return fastEmit_ISD_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
5343 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
5344 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
5345 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
5346 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
5347 default: return Register();
5348 }
5349}
5350
5351// FastEmit functions for ISD::FMINIMUM.
5352
5353Register fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5354 if (RetVT.SimpleTy != MVT::v4f16)
5355 return Register();
5356 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5357 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINhd, RC: &ARM::DPRRegClass, Op0, Op1);
5358 }
5359 return Register();
5360}
5361
5362Register fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5363 if (RetVT.SimpleTy != MVT::v8f16)
5364 return Register();
5365 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5366 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINhq, RC: &ARM::QPRRegClass, Op0, Op1);
5367 }
5368 return Register();
5369}
5370
5371Register fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5372 if (RetVT.SimpleTy != MVT::v2f32)
5373 return Register();
5374 if ((Subtarget->hasNEON())) {
5375 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINfd, RC: &ARM::DPRRegClass, Op0, Op1);
5376 }
5377 return Register();
5378}
5379
5380Register fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5381 if (RetVT.SimpleTy != MVT::v4f32)
5382 return Register();
5383 if ((Subtarget->hasNEON())) {
5384 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINfq, RC: &ARM::QPRRegClass, Op0, Op1);
5385 }
5386 return Register();
5387}
5388
5389Register fastEmit_ISD_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5390 switch (VT.SimpleTy) {
5391 case MVT::v4f16: return fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
5392 case MVT::v8f16: return fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
5393 case MVT::v2f32: return fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
5394 case MVT::v4f32: return fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
5395 default: return Register();
5396 }
5397}
5398
5399// FastEmit functions for ISD::FMINNUM.
5400
5401Register fastEmit_ISD_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5402 if (RetVT.SimpleTy != MVT::f16)
5403 return Register();
5404 if ((Subtarget->hasFullFP16())) {
5405 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMH, RC: &ARM::HPRRegClass, Op0, Op1);
5406 }
5407 return Register();
5408}
5409
5410Register fastEmit_ISD_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5411 if (RetVT.SimpleTy != MVT::f32)
5412 return Register();
5413 if ((Subtarget->hasFPARMv8Base())) {
5414 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMS, RC: &ARM::SPRRegClass, Op0, Op1);
5415 }
5416 return Register();
5417}
5418
5419Register fastEmit_ISD_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5420 if (RetVT.SimpleTy != MVT::f64)
5421 return Register();
5422 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
5423 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMD, RC: &ARM::DPRRegClass, Op0, Op1);
5424 }
5425 return Register();
5426}
5427
5428Register fastEmit_ISD_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5429 if (RetVT.SimpleTy != MVT::v4f16)
5430 return Register();
5431 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5432 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNDh, RC: &ARM::DPRRegClass, Op0, Op1);
5433 }
5434 return Register();
5435}
5436
5437Register fastEmit_ISD_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5438 if (RetVT.SimpleTy != MVT::v8f16)
5439 return Register();
5440 if ((Subtarget->hasMVEFloatOps())) {
5441 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINNMf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5442 }
5443 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5444 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNQh, RC: &ARM::QPRRegClass, Op0, Op1);
5445 }
5446 return Register();
5447}
5448
5449Register fastEmit_ISD_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5450 if (RetVT.SimpleTy != MVT::v2f32)
5451 return Register();
5452 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
5453 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNDf, RC: &ARM::DPRRegClass, Op0, Op1);
5454 }
5455 return Register();
5456}
5457
5458Register fastEmit_ISD_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5459 if (RetVT.SimpleTy != MVT::v4f32)
5460 return Register();
5461 if ((Subtarget->hasMVEFloatOps())) {
5462 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINNMf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5463 }
5464 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
5465 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNQf, RC: &ARM::QPRRegClass, Op0, Op1);
5466 }
5467 return Register();
5468}
5469
5470Register fastEmit_ISD_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5471 switch (VT.SimpleTy) {
5472 case MVT::f16: return fastEmit_ISD_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
5473 case MVT::f32: return fastEmit_ISD_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
5474 case MVT::f64: return fastEmit_ISD_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
5475 case MVT::v4f16: return fastEmit_ISD_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
5476 case MVT::v8f16: return fastEmit_ISD_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
5477 case MVT::v2f32: return fastEmit_ISD_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
5478 case MVT::v4f32: return fastEmit_ISD_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
5479 default: return Register();
5480 }
5481}
5482
5483// FastEmit functions for ISD::FMUL.
5484
5485Register fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5486 if (RetVT.SimpleTy != MVT::f16)
5487 return Register();
5488 if ((Subtarget->hasFullFP16())) {
5489 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULH, RC: &ARM::HPRRegClass, Op0, Op1);
5490 }
5491 return Register();
5492}
5493
5494Register fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5495 if (RetVT.SimpleTy != MVT::f32)
5496 return Register();
5497 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
5498 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULS, RC: &ARM::SPRRegClass, Op0, Op1);
5499 }
5500 return Register();
5501}
5502
5503Register fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5504 if (RetVT.SimpleTy != MVT::f64)
5505 return Register();
5506 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
5507 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULD, RC: &ARM::DPRRegClass, Op0, Op1);
5508 }
5509 return Register();
5510}
5511
5512Register fastEmit_ISD_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5513 if (RetVT.SimpleTy != MVT::v4f16)
5514 return Register();
5515 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5516 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULhd, RC: &ARM::DPRRegClass, Op0, Op1);
5517 }
5518 return Register();
5519}
5520
5521Register fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5522 if (RetVT.SimpleTy != MVT::v8f16)
5523 return Register();
5524 if ((Subtarget->hasMVEFloatOps())) {
5525 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5526 }
5527 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5528 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULhq, RC: &ARM::QPRRegClass, Op0, Op1);
5529 }
5530 return Register();
5531}
5532
5533Register fastEmit_ISD_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5534 if (RetVT.SimpleTy != MVT::v2f32)
5535 return Register();
5536 if ((Subtarget->hasNEON())) {
5537 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULfd, RC: &ARM::DPRRegClass, Op0, Op1);
5538 }
5539 return Register();
5540}
5541
5542Register fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5543 if (RetVT.SimpleTy != MVT::v4f32)
5544 return Register();
5545 if ((Subtarget->hasMVEFloatOps())) {
5546 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5547 }
5548 if ((Subtarget->hasNEON())) {
5549 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULfq, RC: &ARM::QPRRegClass, Op0, Op1);
5550 }
5551 return Register();
5552}
5553
5554Register fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5555 switch (VT.SimpleTy) {
5556 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
5557 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
5558 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
5559 case MVT::v4f16: return fastEmit_ISD_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
5560 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
5561 case MVT::v2f32: return fastEmit_ISD_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
5562 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
5563 default: return Register();
5564 }
5565}
5566
5567// FastEmit functions for ISD::FSUB.
5568
5569Register fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5570 if (RetVT.SimpleTy != MVT::f16)
5571 return Register();
5572 if ((Subtarget->hasFullFP16())) {
5573 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBH, RC: &ARM::HPRRegClass, Op0, Op1);
5574 }
5575 return Register();
5576}
5577
5578Register fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5579 if (RetVT.SimpleTy != MVT::f32)
5580 return Register();
5581 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
5582 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBS, RC: &ARM::SPRRegClass, Op0, Op1);
5583 }
5584 return Register();
5585}
5586
5587Register fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5588 if (RetVT.SimpleTy != MVT::f64)
5589 return Register();
5590 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
5591 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBD, RC: &ARM::DPRRegClass, Op0, Op1);
5592 }
5593 return Register();
5594}
5595
5596Register fastEmit_ISD_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5597 if (RetVT.SimpleTy != MVT::v4f16)
5598 return Register();
5599 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5600 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBhd, RC: &ARM::DPRRegClass, Op0, Op1);
5601 }
5602 return Register();
5603}
5604
5605Register fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5606 if (RetVT.SimpleTy != MVT::v8f16)
5607 return Register();
5608 if ((Subtarget->hasMVEFloatOps())) {
5609 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5610 }
5611 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5612 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBhq, RC: &ARM::QPRRegClass, Op0, Op1);
5613 }
5614 return Register();
5615}
5616
5617Register fastEmit_ISD_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5618 if (RetVT.SimpleTy != MVT::v2f32)
5619 return Register();
5620 if ((Subtarget->hasNEON())) {
5621 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBfd, RC: &ARM::DPRRegClass, Op0, Op1);
5622 }
5623 return Register();
5624}
5625
5626Register fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5627 if (RetVT.SimpleTy != MVT::v4f32)
5628 return Register();
5629 if ((Subtarget->hasMVEFloatOps())) {
5630 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5631 }
5632 if ((Subtarget->hasNEON())) {
5633 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBfq, RC: &ARM::QPRRegClass, Op0, Op1);
5634 }
5635 return Register();
5636}
5637
5638Register fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5639 switch (VT.SimpleTy) {
5640 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
5641 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
5642 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
5643 case MVT::v4f16: return fastEmit_ISD_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
5644 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
5645 case MVT::v2f32: return fastEmit_ISD_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
5646 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
5647 default: return Register();
5648 }
5649}
5650
5651// FastEmit functions for ISD::MUL.
5652
5653Register fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5654 if (RetVT.SimpleTy != MVT::i32)
5655 return Register();
5656 if ((Subtarget->isThumb2())) {
5657 return fastEmitInst_rr(MachineInstOpcode: ARM::t2MUL, RC: &ARM::rGPRRegClass, Op0, Op1);
5658 }
5659 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5660 return fastEmitInst_rr(MachineInstOpcode: ARM::tMUL, RC: &ARM::tGPRRegClass, Op0, Op1);
5661 }
5662 if ((!Subtarget->isThumb()) && (!Subtarget->hasV6Ops()) && (Subtarget->useMulOps())) {
5663 return fastEmitInst_rr(MachineInstOpcode: ARM::MULv5, RC: &ARM::GPRnopcRegClass, Op0, Op1);
5664 }
5665 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
5666 return fastEmitInst_rr(MachineInstOpcode: ARM::MUL, RC: &ARM::GPRnopcRegClass, Op0, Op1);
5667 }
5668 return Register();
5669}
5670
5671Register fastEmit_ISD_MUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
5672 if (RetVT.SimpleTy != MVT::v8i8)
5673 return Register();
5674 if ((Subtarget->hasNEON())) {
5675 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
5676 }
5677 return Register();
5678}
5679
5680Register fastEmit_ISD_MUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5681 if (RetVT.SimpleTy != MVT::v16i8)
5682 return Register();
5683 if ((Subtarget->hasMVEIntegerOps())) {
5684 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULi8, RC: &ARM::MQPRRegClass, Op0, Op1);
5685 }
5686 if ((Subtarget->hasNEON())) {
5687 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
5688 }
5689 return Register();
5690}
5691
5692Register fastEmit_ISD_MUL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
5693 if (RetVT.SimpleTy != MVT::v4i16)
5694 return Register();
5695 if ((Subtarget->hasNEON())) {
5696 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
5697 }
5698 return Register();
5699}
5700
5701Register fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5702 if (RetVT.SimpleTy != MVT::v8i16)
5703 return Register();
5704 if ((Subtarget->hasMVEIntegerOps())) {
5705 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULi16, RC: &ARM::MQPRRegClass, Op0, Op1);
5706 }
5707 if ((Subtarget->hasNEON())) {
5708 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
5709 }
5710 return Register();
5711}
5712
5713Register fastEmit_ISD_MUL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
5714 if (RetVT.SimpleTy != MVT::v2i32)
5715 return Register();
5716 if ((Subtarget->hasNEON())) {
5717 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
5718 }
5719 return Register();
5720}
5721
5722Register fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5723 if (RetVT.SimpleTy != MVT::v4i32)
5724 return Register();
5725 if ((Subtarget->hasMVEIntegerOps())) {
5726 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULi32, RC: &ARM::MQPRRegClass, Op0, Op1);
5727 }
5728 if ((Subtarget->hasNEON())) {
5729 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
5730 }
5731 return Register();
5732}
5733
5734Register fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5735 switch (VT.SimpleTy) {
5736 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op1);
5737 case MVT::v8i8: return fastEmit_ISD_MUL_MVT_v8i8_rr(RetVT, Op0, Op1);
5738 case MVT::v16i8: return fastEmit_ISD_MUL_MVT_v16i8_rr(RetVT, Op0, Op1);
5739 case MVT::v4i16: return fastEmit_ISD_MUL_MVT_v4i16_rr(RetVT, Op0, Op1);
5740 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
5741 case MVT::v2i32: return fastEmit_ISD_MUL_MVT_v2i32_rr(RetVT, Op0, Op1);
5742 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
5743 default: return Register();
5744 }
5745}
5746
5747// FastEmit functions for ISD::MULHS.
5748
5749Register fastEmit_ISD_MULHS_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5750 if (RetVT.SimpleTy != MVT::i32)
5751 return Register();
5752 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
5753 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SMMUL, RC: &ARM::rGPRRegClass, Op0, Op1);
5754 }
5755 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
5756 return fastEmitInst_rr(MachineInstOpcode: ARM::SMMUL, RC: &ARM::GPRRegClass, Op0, Op1);
5757 }
5758 return Register();
5759}
5760
5761Register fastEmit_ISD_MULHS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5762 if (RetVT.SimpleTy != MVT::v16i8)
5763 return Register();
5764 if ((Subtarget->hasMVEIntegerOps())) {
5765 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHs8, RC: &ARM::MQPRRegClass, Op0, Op1);
5766 }
5767 return Register();
5768}
5769
5770Register fastEmit_ISD_MULHS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5771 if (RetVT.SimpleTy != MVT::v8i16)
5772 return Register();
5773 if ((Subtarget->hasMVEIntegerOps())) {
5774 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHs16, RC: &ARM::MQPRRegClass, Op0, Op1);
5775 }
5776 return Register();
5777}
5778
5779Register fastEmit_ISD_MULHS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5780 if (RetVT.SimpleTy != MVT::v4i32)
5781 return Register();
5782 if ((Subtarget->hasMVEIntegerOps())) {
5783 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHs32, RC: &ARM::MQPRRegClass, Op0, Op1);
5784 }
5785 return Register();
5786}
5787
5788Register fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5789 switch (VT.SimpleTy) {
5790 case MVT::i32: return fastEmit_ISD_MULHS_MVT_i32_rr(RetVT, Op0, Op1);
5791 case MVT::v16i8: return fastEmit_ISD_MULHS_MVT_v16i8_rr(RetVT, Op0, Op1);
5792 case MVT::v8i16: return fastEmit_ISD_MULHS_MVT_v8i16_rr(RetVT, Op0, Op1);
5793 case MVT::v4i32: return fastEmit_ISD_MULHS_MVT_v4i32_rr(RetVT, Op0, Op1);
5794 default: return Register();
5795 }
5796}
5797
5798// FastEmit functions for ISD::MULHU.
5799
5800Register fastEmit_ISD_MULHU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5801 if (RetVT.SimpleTy != MVT::v16i8)
5802 return Register();
5803 if ((Subtarget->hasMVEIntegerOps())) {
5804 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHu8, RC: &ARM::MQPRRegClass, Op0, Op1);
5805 }
5806 return Register();
5807}
5808
5809Register fastEmit_ISD_MULHU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5810 if (RetVT.SimpleTy != MVT::v8i16)
5811 return Register();
5812 if ((Subtarget->hasMVEIntegerOps())) {
5813 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHu16, RC: &ARM::MQPRRegClass, Op0, Op1);
5814 }
5815 return Register();
5816}
5817
5818Register fastEmit_ISD_MULHU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5819 if (RetVT.SimpleTy != MVT::v4i32)
5820 return Register();
5821 if ((Subtarget->hasMVEIntegerOps())) {
5822 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHu32, RC: &ARM::MQPRRegClass, Op0, Op1);
5823 }
5824 return Register();
5825}
5826
5827Register fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5828 switch (VT.SimpleTy) {
5829 case MVT::v16i8: return fastEmit_ISD_MULHU_MVT_v16i8_rr(RetVT, Op0, Op1);
5830 case MVT::v8i16: return fastEmit_ISD_MULHU_MVT_v8i16_rr(RetVT, Op0, Op1);
5831 case MVT::v4i32: return fastEmit_ISD_MULHU_MVT_v4i32_rr(RetVT, Op0, Op1);
5832 default: return Register();
5833 }
5834}
5835
5836// FastEmit functions for ISD::OR.
5837
5838Register fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5839 if (RetVT.SimpleTy != MVT::i32)
5840 return Register();
5841 if ((Subtarget->isThumb2())) {
5842 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ORRrr, RC: &ARM::rGPRRegClass, Op0, Op1);
5843 }
5844 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5845 return fastEmitInst_rr(MachineInstOpcode: ARM::tORR, RC: &ARM::tGPRRegClass, Op0, Op1);
5846 }
5847 if ((!Subtarget->isThumb())) {
5848 return fastEmitInst_rr(MachineInstOpcode: ARM::ORRrr, RC: &ARM::GPRRegClass, Op0, Op1);
5849 }
5850 return Register();
5851}
5852
5853Register fastEmit_ISD_OR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
5854 if (RetVT.SimpleTy != MVT::v8i8)
5855 return Register();
5856 if ((Subtarget->hasNEON())) {
5857 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
5858 }
5859 return Register();
5860}
5861
5862Register fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5863 if (RetVT.SimpleTy != MVT::v16i8)
5864 return Register();
5865 if ((Subtarget->hasMVEIntegerOps())) {
5866 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
5867 }
5868 if ((Subtarget->hasNEON())) {
5869 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
5870 }
5871 return Register();
5872}
5873
5874Register fastEmit_ISD_OR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
5875 if (RetVT.SimpleTy != MVT::v4i16)
5876 return Register();
5877 if ((Subtarget->hasNEON())) {
5878 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
5879 }
5880 return Register();
5881}
5882
5883Register fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5884 if (RetVT.SimpleTy != MVT::v8i16)
5885 return Register();
5886 if ((Subtarget->hasMVEIntegerOps())) {
5887 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
5888 }
5889 if ((Subtarget->hasNEON())) {
5890 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
5891 }
5892 return Register();
5893}
5894
5895Register fastEmit_ISD_OR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
5896 if (RetVT.SimpleTy != MVT::v2i32)
5897 return Register();
5898 if ((Subtarget->hasNEON())) {
5899 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
5900 }
5901 return Register();
5902}
5903
5904Register fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5905 if (RetVT.SimpleTy != MVT::v4i32)
5906 return Register();
5907 if ((Subtarget->hasMVEIntegerOps())) {
5908 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
5909 }
5910 if ((Subtarget->hasNEON())) {
5911 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
5912 }
5913 return Register();
5914}
5915
5916Register fastEmit_ISD_OR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
5917 if (RetVT.SimpleTy != MVT::v1i64)
5918 return Register();
5919 if ((Subtarget->hasNEON())) {
5920 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
5921 }
5922 return Register();
5923}
5924
5925Register fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
5926 if (RetVT.SimpleTy != MVT::v2i64)
5927 return Register();
5928 if ((Subtarget->hasMVEIntegerOps())) {
5929 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
5930 }
5931 if ((Subtarget->hasNEON())) {
5932 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
5933 }
5934 return Register();
5935}
5936
5937Register fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5938 switch (VT.SimpleTy) {
5939 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
5940 case MVT::v8i8: return fastEmit_ISD_OR_MVT_v8i8_rr(RetVT, Op0, Op1);
5941 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
5942 case MVT::v4i16: return fastEmit_ISD_OR_MVT_v4i16_rr(RetVT, Op0, Op1);
5943 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
5944 case MVT::v2i32: return fastEmit_ISD_OR_MVT_v2i32_rr(RetVT, Op0, Op1);
5945 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
5946 case MVT::v1i64: return fastEmit_ISD_OR_MVT_v1i64_rr(RetVT, Op0, Op1);
5947 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
5948 default: return Register();
5949 }
5950}
5951
5952// FastEmit functions for ISD::ROTR.
5953
5954Register fastEmit_ISD_ROTR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5955 if (RetVT.SimpleTy != MVT::i32)
5956 return Register();
5957 if ((Subtarget->isThumb2())) {
5958 return fastEmitInst_rr(MachineInstOpcode: ARM::t2RORrr, RC: &ARM::rGPRRegClass, Op0, Op1);
5959 }
5960 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5961 return fastEmitInst_rr(MachineInstOpcode: ARM::tROR, RC: &ARM::tGPRRegClass, Op0, Op1);
5962 }
5963 return Register();
5964}
5965
5966Register fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5967 switch (VT.SimpleTy) {
5968 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_rr(RetVT, Op0, Op1);
5969 default: return Register();
5970 }
5971}
5972
5973// FastEmit functions for ISD::SADDSAT.
5974
5975Register fastEmit_ISD_SADDSAT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5976 if (RetVT.SimpleTy != MVT::i32)
5977 return Register();
5978 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
5979 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QADD, RC: &ARM::rGPRRegClass, Op0, Op1);
5980 }
5981 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
5982 return fastEmitInst_rr(MachineInstOpcode: ARM::QADD, RC: &ARM::GPRnopcRegClass, Op0, Op1);
5983 }
5984 return Register();
5985}
5986
5987Register fastEmit_ISD_SADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
5988 if (RetVT.SimpleTy != MVT::v8i8)
5989 return Register();
5990 if ((Subtarget->hasNEON())) {
5991 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
5992 }
5993 return Register();
5994}
5995
5996Register fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5997 if (RetVT.SimpleTy != MVT::v16i8)
5998 return Register();
5999 if ((Subtarget->hasMVEIntegerOps())) {
6000 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
6001 }
6002 if ((Subtarget->hasNEON())) {
6003 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6004 }
6005 return Register();
6006}
6007
6008Register fastEmit_ISD_SADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6009 if (RetVT.SimpleTy != MVT::v4i16)
6010 return Register();
6011 if ((Subtarget->hasNEON())) {
6012 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6013 }
6014 return Register();
6015}
6016
6017Register fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6018 if (RetVT.SimpleTy != MVT::v8i16)
6019 return Register();
6020 if ((Subtarget->hasMVEIntegerOps())) {
6021 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
6022 }
6023 if ((Subtarget->hasNEON())) {
6024 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6025 }
6026 return Register();
6027}
6028
6029Register fastEmit_ISD_SADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6030 if (RetVT.SimpleTy != MVT::v2i32)
6031 return Register();
6032 if ((Subtarget->hasNEON())) {
6033 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6034 }
6035 return Register();
6036}
6037
6038Register fastEmit_ISD_SADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6039 if (RetVT.SimpleTy != MVT::v4i32)
6040 return Register();
6041 if ((Subtarget->hasMVEIntegerOps())) {
6042 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
6043 }
6044 if ((Subtarget->hasNEON())) {
6045 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6046 }
6047 return Register();
6048}
6049
6050Register fastEmit_ISD_SADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
6051 if (RetVT.SimpleTy != MVT::v1i64)
6052 return Register();
6053 if ((Subtarget->hasNEON())) {
6054 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
6055 }
6056 return Register();
6057}
6058
6059Register fastEmit_ISD_SADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6060 if (RetVT.SimpleTy != MVT::v2i64)
6061 return Register();
6062 if ((Subtarget->hasNEON())) {
6063 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
6064 }
6065 return Register();
6066}
6067
6068Register fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6069 switch (VT.SimpleTy) {
6070 case MVT::i32: return fastEmit_ISD_SADDSAT_MVT_i32_rr(RetVT, Op0, Op1);
6071 case MVT::v8i8: return fastEmit_ISD_SADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
6072 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
6073 case MVT::v4i16: return fastEmit_ISD_SADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
6074 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
6075 case MVT::v2i32: return fastEmit_ISD_SADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
6076 case MVT::v4i32: return fastEmit_ISD_SADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
6077 case MVT::v1i64: return fastEmit_ISD_SADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
6078 case MVT::v2i64: return fastEmit_ISD_SADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
6079 default: return Register();
6080 }
6081}
6082
6083// FastEmit functions for ISD::SDIV.
6084
6085Register fastEmit_ISD_SDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6086 if (RetVT.SimpleTy != MVT::i32)
6087 return Register();
6088 if ((Subtarget->hasDivideInThumbMode()) && (Subtarget->hasV8MBaselineOps()) && (Subtarget->isThumb())) {
6089 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SDIV, RC: &ARM::rGPRRegClass, Op0, Op1);
6090 }
6091 if ((Subtarget->hasDivideInARMMode()) && (!Subtarget->isThumb())) {
6092 return fastEmitInst_rr(MachineInstOpcode: ARM::SDIV, RC: &ARM::GPRRegClass, Op0, Op1);
6093 }
6094 return Register();
6095}
6096
6097Register fastEmit_ISD_SDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6098 switch (VT.SimpleTy) {
6099 case MVT::i32: return fastEmit_ISD_SDIV_MVT_i32_rr(RetVT, Op0, Op1);
6100 default: return Register();
6101 }
6102}
6103
6104// FastEmit functions for ISD::SHL.
6105
6106Register fastEmit_ISD_SHL_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6107 if (RetVT.SimpleTy != MVT::i32)
6108 return Register();
6109 if ((Subtarget->isThumb2())) {
6110 return fastEmitInst_rr(MachineInstOpcode: ARM::t2LSLrr, RC: &ARM::rGPRRegClass, Op0, Op1);
6111 }
6112 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6113 return fastEmitInst_rr(MachineInstOpcode: ARM::tLSLrr, RC: &ARM::tGPRRegClass, Op0, Op1);
6114 }
6115 return Register();
6116}
6117
6118Register fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6119 switch (VT.SimpleTy) {
6120 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_rr(RetVT, Op0, Op1);
6121 default: return Register();
6122 }
6123}
6124
6125// FastEmit functions for ISD::SMAX.
6126
6127Register fastEmit_ISD_SMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6128 if (RetVT.SimpleTy != MVT::v8i8)
6129 return Register();
6130 if ((Subtarget->hasNEON())) {
6131 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6132 }
6133 return Register();
6134}
6135
6136Register fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6137 if (RetVT.SimpleTy != MVT::v16i8)
6138 return Register();
6139 if ((Subtarget->hasMVEIntegerOps())) {
6140 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXs8, RC: &ARM::MQPRRegClass, Op0, Op1);
6141 }
6142 if ((Subtarget->hasNEON())) {
6143 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6144 }
6145 return Register();
6146}
6147
6148Register fastEmit_ISD_SMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6149 if (RetVT.SimpleTy != MVT::v4i16)
6150 return Register();
6151 if ((Subtarget->hasNEON())) {
6152 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6153 }
6154 return Register();
6155}
6156
6157Register fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6158 if (RetVT.SimpleTy != MVT::v8i16)
6159 return Register();
6160 if ((Subtarget->hasMVEIntegerOps())) {
6161 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXs16, RC: &ARM::MQPRRegClass, Op0, Op1);
6162 }
6163 if ((Subtarget->hasNEON())) {
6164 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6165 }
6166 return Register();
6167}
6168
6169Register fastEmit_ISD_SMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6170 if (RetVT.SimpleTy != MVT::v2i32)
6171 return Register();
6172 if ((Subtarget->hasNEON())) {
6173 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6174 }
6175 return Register();
6176}
6177
6178Register fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6179 if (RetVT.SimpleTy != MVT::v4i32)
6180 return Register();
6181 if ((Subtarget->hasMVEIntegerOps())) {
6182 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXs32, RC: &ARM::MQPRRegClass, Op0, Op1);
6183 }
6184 if ((Subtarget->hasNEON())) {
6185 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6186 }
6187 return Register();
6188}
6189
6190Register fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6191 switch (VT.SimpleTy) {
6192 case MVT::v8i8: return fastEmit_ISD_SMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
6193 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
6194 case MVT::v4i16: return fastEmit_ISD_SMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
6195 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
6196 case MVT::v2i32: return fastEmit_ISD_SMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
6197 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
6198 default: return Register();
6199 }
6200}
6201
6202// FastEmit functions for ISD::SMIN.
6203
6204Register fastEmit_ISD_SMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6205 if (RetVT.SimpleTy != MVT::v8i8)
6206 return Register();
6207 if ((Subtarget->hasNEON())) {
6208 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6209 }
6210 return Register();
6211}
6212
6213Register fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6214 if (RetVT.SimpleTy != MVT::v16i8)
6215 return Register();
6216 if ((Subtarget->hasMVEIntegerOps())) {
6217 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINs8, RC: &ARM::MQPRRegClass, Op0, Op1);
6218 }
6219 if ((Subtarget->hasNEON())) {
6220 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6221 }
6222 return Register();
6223}
6224
6225Register fastEmit_ISD_SMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6226 if (RetVT.SimpleTy != MVT::v4i16)
6227 return Register();
6228 if ((Subtarget->hasNEON())) {
6229 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6230 }
6231 return Register();
6232}
6233
6234Register fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6235 if (RetVT.SimpleTy != MVT::v8i16)
6236 return Register();
6237 if ((Subtarget->hasMVEIntegerOps())) {
6238 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINs16, RC: &ARM::MQPRRegClass, Op0, Op1);
6239 }
6240 if ((Subtarget->hasNEON())) {
6241 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6242 }
6243 return Register();
6244}
6245
6246Register fastEmit_ISD_SMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6247 if (RetVT.SimpleTy != MVT::v2i32)
6248 return Register();
6249 if ((Subtarget->hasNEON())) {
6250 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6251 }
6252 return Register();
6253}
6254
6255Register fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6256 if (RetVT.SimpleTy != MVT::v4i32)
6257 return Register();
6258 if ((Subtarget->hasMVEIntegerOps())) {
6259 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINs32, RC: &ARM::MQPRRegClass, Op0, Op1);
6260 }
6261 if ((Subtarget->hasNEON())) {
6262 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6263 }
6264 return Register();
6265}
6266
6267Register fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6268 switch (VT.SimpleTy) {
6269 case MVT::v8i8: return fastEmit_ISD_SMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
6270 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
6271 case MVT::v4i16: return fastEmit_ISD_SMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
6272 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
6273 case MVT::v2i32: return fastEmit_ISD_SMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
6274 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
6275 default: return Register();
6276 }
6277}
6278
6279// FastEmit functions for ISD::SRA.
6280
6281Register fastEmit_ISD_SRA_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6282 if (RetVT.SimpleTy != MVT::i32)
6283 return Register();
6284 if ((Subtarget->isThumb2())) {
6285 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ASRrr, RC: &ARM::rGPRRegClass, Op0, Op1);
6286 }
6287 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6288 return fastEmitInst_rr(MachineInstOpcode: ARM::tASRrr, RC: &ARM::tGPRRegClass, Op0, Op1);
6289 }
6290 return Register();
6291}
6292
6293Register fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6294 switch (VT.SimpleTy) {
6295 case MVT::i32: return fastEmit_ISD_SRA_MVT_i32_rr(RetVT, Op0, Op1);
6296 default: return Register();
6297 }
6298}
6299
6300// FastEmit functions for ISD::SRL.
6301
6302Register fastEmit_ISD_SRL_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6303 if (RetVT.SimpleTy != MVT::i32)
6304 return Register();
6305 if ((Subtarget->isThumb2())) {
6306 return fastEmitInst_rr(MachineInstOpcode: ARM::t2LSRrr, RC: &ARM::rGPRRegClass, Op0, Op1);
6307 }
6308 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6309 return fastEmitInst_rr(MachineInstOpcode: ARM::tLSRrr, RC: &ARM::tGPRRegClass, Op0, Op1);
6310 }
6311 return Register();
6312}
6313
6314Register fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6315 switch (VT.SimpleTy) {
6316 case MVT::i32: return fastEmit_ISD_SRL_MVT_i32_rr(RetVT, Op0, Op1);
6317 default: return Register();
6318 }
6319}
6320
6321// FastEmit functions for ISD::SSUBSAT.
6322
6323Register fastEmit_ISD_SSUBSAT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6324 if (RetVT.SimpleTy != MVT::i32)
6325 return Register();
6326 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
6327 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QSUB, RC: &ARM::rGPRRegClass, Op0, Op1);
6328 }
6329 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
6330 return fastEmitInst_rr(MachineInstOpcode: ARM::QSUB, RC: &ARM::GPRnopcRegClass, Op0, Op1);
6331 }
6332 return Register();
6333}
6334
6335Register fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6336 if (RetVT.SimpleTy != MVT::v8i8)
6337 return Register();
6338 if ((Subtarget->hasNEON())) {
6339 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6340 }
6341 return Register();
6342}
6343
6344Register fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6345 if (RetVT.SimpleTy != MVT::v16i8)
6346 return Register();
6347 if ((Subtarget->hasMVEIntegerOps())) {
6348 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBs8, RC: &ARM::MQPRRegClass, Op0, Op1);
6349 }
6350 if ((Subtarget->hasNEON())) {
6351 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6352 }
6353 return Register();
6354}
6355
6356Register fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6357 if (RetVT.SimpleTy != MVT::v4i16)
6358 return Register();
6359 if ((Subtarget->hasNEON())) {
6360 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6361 }
6362 return Register();
6363}
6364
6365Register fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6366 if (RetVT.SimpleTy != MVT::v8i16)
6367 return Register();
6368 if ((Subtarget->hasMVEIntegerOps())) {
6369 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBs16, RC: &ARM::MQPRRegClass, Op0, Op1);
6370 }
6371 if ((Subtarget->hasNEON())) {
6372 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6373 }
6374 return Register();
6375}
6376
6377Register fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6378 if (RetVT.SimpleTy != MVT::v2i32)
6379 return Register();
6380 if ((Subtarget->hasNEON())) {
6381 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6382 }
6383 return Register();
6384}
6385
6386Register fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6387 if (RetVT.SimpleTy != MVT::v4i32)
6388 return Register();
6389 if ((Subtarget->hasMVEIntegerOps())) {
6390 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBs32, RC: &ARM::MQPRRegClass, Op0, Op1);
6391 }
6392 if ((Subtarget->hasNEON())) {
6393 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6394 }
6395 return Register();
6396}
6397
6398Register fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
6399 if (RetVT.SimpleTy != MVT::v1i64)
6400 return Register();
6401 if ((Subtarget->hasNEON())) {
6402 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
6403 }
6404 return Register();
6405}
6406
6407Register fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6408 if (RetVT.SimpleTy != MVT::v2i64)
6409 return Register();
6410 if ((Subtarget->hasNEON())) {
6411 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
6412 }
6413 return Register();
6414}
6415
6416Register fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6417 switch (VT.SimpleTy) {
6418 case MVT::i32: return fastEmit_ISD_SSUBSAT_MVT_i32_rr(RetVT, Op0, Op1);
6419 case MVT::v8i8: return fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
6420 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
6421 case MVT::v4i16: return fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
6422 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
6423 case MVT::v2i32: return fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
6424 case MVT::v4i32: return fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
6425 case MVT::v1i64: return fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
6426 case MVT::v2i64: return fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
6427 default: return Register();
6428 }
6429}
6430
6431// FastEmit functions for ISD::STRICT_FADD.
6432
6433Register fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6434 if (RetVT.SimpleTy != MVT::f16)
6435 return Register();
6436 if ((Subtarget->hasFullFP16())) {
6437 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDH, RC: &ARM::HPRRegClass, Op0, Op1);
6438 }
6439 return Register();
6440}
6441
6442Register fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6443 if (RetVT.SimpleTy != MVT::f32)
6444 return Register();
6445 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
6446 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDS, RC: &ARM::SPRRegClass, Op0, Op1);
6447 }
6448 return Register();
6449}
6450
6451Register fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6452 if (RetVT.SimpleTy != MVT::f64)
6453 return Register();
6454 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
6455 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDD, RC: &ARM::DPRRegClass, Op0, Op1);
6456 }
6457 return Register();
6458}
6459
6460Register fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6461 switch (VT.SimpleTy) {
6462 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1);
6463 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1);
6464 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1);
6465 default: return Register();
6466 }
6467}
6468
6469// FastEmit functions for ISD::STRICT_FDIV.
6470
6471Register fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6472 if (RetVT.SimpleTy != MVT::f16)
6473 return Register();
6474 if ((Subtarget->hasFullFP16())) {
6475 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVH, RC: &ARM::HPRRegClass, Op0, Op1);
6476 }
6477 return Register();
6478}
6479
6480Register fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6481 if (RetVT.SimpleTy != MVT::f32)
6482 return Register();
6483 if ((Subtarget->hasVFP2Base())) {
6484 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVS, RC: &ARM::SPRRegClass, Op0, Op1);
6485 }
6486 return Register();
6487}
6488
6489Register fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6490 if (RetVT.SimpleTy != MVT::f64)
6491 return Register();
6492 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
6493 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVD, RC: &ARM::DPRRegClass, Op0, Op1);
6494 }
6495 return Register();
6496}
6497
6498Register fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6499 switch (VT.SimpleTy) {
6500 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
6501 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
6502 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
6503 default: return Register();
6504 }
6505}
6506
6507// FastEmit functions for ISD::STRICT_FMAXNUM.
6508
6509Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6510 if (RetVT.SimpleTy != MVT::f16)
6511 return Register();
6512 if ((Subtarget->hasFullFP16())) {
6513 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMH, RC: &ARM::HPRRegClass, Op0, Op1);
6514 }
6515 return Register();
6516}
6517
6518Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6519 if (RetVT.SimpleTy != MVT::f32)
6520 return Register();
6521 if ((Subtarget->hasFPARMv8Base())) {
6522 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMS, RC: &ARM::SPRRegClass, Op0, Op1);
6523 }
6524 return Register();
6525}
6526
6527Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6528 if (RetVT.SimpleTy != MVT::f64)
6529 return Register();
6530 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
6531 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMD, RC: &ARM::DPRRegClass, Op0, Op1);
6532 }
6533 return Register();
6534}
6535
6536Register fastEmit_ISD_STRICT_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6537 switch (VT.SimpleTy) {
6538 case MVT::f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
6539 case MVT::f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
6540 case MVT::f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
6541 default: return Register();
6542 }
6543}
6544
6545// FastEmit functions for ISD::STRICT_FMINNUM.
6546
6547Register fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6548 if (RetVT.SimpleTy != MVT::f16)
6549 return Register();
6550 if ((Subtarget->hasFullFP16())) {
6551 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMH, RC: &ARM::HPRRegClass, Op0, Op1);
6552 }
6553 return Register();
6554}
6555
6556Register fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6557 if (RetVT.SimpleTy != MVT::f32)
6558 return Register();
6559 if ((Subtarget->hasFPARMv8Base())) {
6560 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMS, RC: &ARM::SPRRegClass, Op0, Op1);
6561 }
6562 return Register();
6563}
6564
6565Register fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6566 if (RetVT.SimpleTy != MVT::f64)
6567 return Register();
6568 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
6569 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMD, RC: &ARM::DPRRegClass, Op0, Op1);
6570 }
6571 return Register();
6572}
6573
6574Register fastEmit_ISD_STRICT_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6575 switch (VT.SimpleTy) {
6576 case MVT::f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
6577 case MVT::f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
6578 case MVT::f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
6579 default: return Register();
6580 }
6581}
6582
6583// FastEmit functions for ISD::STRICT_FMUL.
6584
6585Register fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6586 if (RetVT.SimpleTy != MVT::f16)
6587 return Register();
6588 if ((Subtarget->hasFullFP16())) {
6589 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULH, RC: &ARM::HPRRegClass, Op0, Op1);
6590 }
6591 return Register();
6592}
6593
6594Register fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6595 if (RetVT.SimpleTy != MVT::f32)
6596 return Register();
6597 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
6598 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULS, RC: &ARM::SPRRegClass, Op0, Op1);
6599 }
6600 return Register();
6601}
6602
6603Register fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6604 if (RetVT.SimpleTy != MVT::f64)
6605 return Register();
6606 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
6607 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULD, RC: &ARM::DPRRegClass, Op0, Op1);
6608 }
6609 return Register();
6610}
6611
6612Register fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6613 switch (VT.SimpleTy) {
6614 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
6615 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
6616 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
6617 default: return Register();
6618 }
6619}
6620
6621// FastEmit functions for ISD::STRICT_FSUB.
6622
6623Register fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6624 if (RetVT.SimpleTy != MVT::f16)
6625 return Register();
6626 if ((Subtarget->hasFullFP16())) {
6627 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBH, RC: &ARM::HPRRegClass, Op0, Op1);
6628 }
6629 return Register();
6630}
6631
6632Register fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6633 if (RetVT.SimpleTy != MVT::f32)
6634 return Register();
6635 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
6636 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBS, RC: &ARM::SPRRegClass, Op0, Op1);
6637 }
6638 return Register();
6639}
6640
6641Register fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6642 if (RetVT.SimpleTy != MVT::f64)
6643 return Register();
6644 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
6645 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBD, RC: &ARM::DPRRegClass, Op0, Op1);
6646 }
6647 return Register();
6648}
6649
6650Register fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6651 switch (VT.SimpleTy) {
6652 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
6653 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
6654 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
6655 default: return Register();
6656 }
6657}
6658
6659// FastEmit functions for ISD::SUB.
6660
6661Register fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6662 if (RetVT.SimpleTy != MVT::i32)
6663 return Register();
6664 if ((Subtarget->isThumb2())) {
6665 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SUBrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
6666 }
6667 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6668 return fastEmitInst_rr(MachineInstOpcode: ARM::tSUBrr, RC: &ARM::tGPRRegClass, Op0, Op1);
6669 }
6670 if ((!Subtarget->isThumb())) {
6671 return fastEmitInst_rr(MachineInstOpcode: ARM::SUBrr, RC: &ARM::GPRRegClass, Op0, Op1);
6672 }
6673 return Register();
6674}
6675
6676Register fastEmit_ISD_SUB_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6677 if (RetVT.SimpleTy != MVT::v8i8)
6678 return Register();
6679 if ((Subtarget->hasNEON())) {
6680 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6681 }
6682 return Register();
6683}
6684
6685Register fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6686 if (RetVT.SimpleTy != MVT::v16i8)
6687 return Register();
6688 if ((Subtarget->hasMVEIntegerOps())) {
6689 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBi8, RC: &ARM::MQPRRegClass, Op0, Op1);
6690 }
6691 if ((Subtarget->hasNEON())) {
6692 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6693 }
6694 return Register();
6695}
6696
6697Register fastEmit_ISD_SUB_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6698 if (RetVT.SimpleTy != MVT::v4i16)
6699 return Register();
6700 if ((Subtarget->hasNEON())) {
6701 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6702 }
6703 return Register();
6704}
6705
6706Register fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6707 if (RetVT.SimpleTy != MVT::v8i16)
6708 return Register();
6709 if ((Subtarget->hasMVEIntegerOps())) {
6710 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBi16, RC: &ARM::MQPRRegClass, Op0, Op1);
6711 }
6712 if ((Subtarget->hasNEON())) {
6713 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6714 }
6715 return Register();
6716}
6717
6718Register fastEmit_ISD_SUB_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6719 if (RetVT.SimpleTy != MVT::v2i32)
6720 return Register();
6721 if ((Subtarget->hasNEON())) {
6722 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6723 }
6724 return Register();
6725}
6726
6727Register fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6728 if (RetVT.SimpleTy != MVT::v4i32)
6729 return Register();
6730 if ((Subtarget->hasMVEIntegerOps())) {
6731 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBi32, RC: &ARM::MQPRRegClass, Op0, Op1);
6732 }
6733 if ((Subtarget->hasNEON())) {
6734 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6735 }
6736 return Register();
6737}
6738
6739Register fastEmit_ISD_SUB_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
6740 if (RetVT.SimpleTy != MVT::v1i64)
6741 return Register();
6742 if ((Subtarget->hasNEON())) {
6743 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
6744 }
6745 return Register();
6746}
6747
6748Register fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6749 if (RetVT.SimpleTy != MVT::v2i64)
6750 return Register();
6751 if ((Subtarget->hasNEON())) {
6752 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
6753 }
6754 return Register();
6755}
6756
6757Register fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6758 switch (VT.SimpleTy) {
6759 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
6760 case MVT::v8i8: return fastEmit_ISD_SUB_MVT_v8i8_rr(RetVT, Op0, Op1);
6761 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
6762 case MVT::v4i16: return fastEmit_ISD_SUB_MVT_v4i16_rr(RetVT, Op0, Op1);
6763 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
6764 case MVT::v2i32: return fastEmit_ISD_SUB_MVT_v2i32_rr(RetVT, Op0, Op1);
6765 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
6766 case MVT::v1i64: return fastEmit_ISD_SUB_MVT_v1i64_rr(RetVT, Op0, Op1);
6767 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
6768 default: return Register();
6769 }
6770}
6771
6772// FastEmit functions for ISD::UADDSAT.
6773
6774Register fastEmit_ISD_UADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6775 if (RetVT.SimpleTy != MVT::v8i8)
6776 return Register();
6777 if ((Subtarget->hasNEON())) {
6778 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6779 }
6780 return Register();
6781}
6782
6783Register fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6784 if (RetVT.SimpleTy != MVT::v16i8)
6785 return Register();
6786 if ((Subtarget->hasMVEIntegerOps())) {
6787 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
6788 }
6789 if ((Subtarget->hasNEON())) {
6790 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6791 }
6792 return Register();
6793}
6794
6795Register fastEmit_ISD_UADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6796 if (RetVT.SimpleTy != MVT::v4i16)
6797 return Register();
6798 if ((Subtarget->hasNEON())) {
6799 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6800 }
6801 return Register();
6802}
6803
6804Register fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6805 if (RetVT.SimpleTy != MVT::v8i16)
6806 return Register();
6807 if ((Subtarget->hasMVEIntegerOps())) {
6808 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
6809 }
6810 if ((Subtarget->hasNEON())) {
6811 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6812 }
6813 return Register();
6814}
6815
6816Register fastEmit_ISD_UADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6817 if (RetVT.SimpleTy != MVT::v2i32)
6818 return Register();
6819 if ((Subtarget->hasNEON())) {
6820 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6821 }
6822 return Register();
6823}
6824
6825Register fastEmit_ISD_UADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6826 if (RetVT.SimpleTy != MVT::v4i32)
6827 return Register();
6828 if ((Subtarget->hasMVEIntegerOps())) {
6829 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
6830 }
6831 if ((Subtarget->hasNEON())) {
6832 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6833 }
6834 return Register();
6835}
6836
6837Register fastEmit_ISD_UADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
6838 if (RetVT.SimpleTy != MVT::v1i64)
6839 return Register();
6840 if ((Subtarget->hasNEON())) {
6841 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
6842 }
6843 return Register();
6844}
6845
6846Register fastEmit_ISD_UADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6847 if (RetVT.SimpleTy != MVT::v2i64)
6848 return Register();
6849 if ((Subtarget->hasNEON())) {
6850 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
6851 }
6852 return Register();
6853}
6854
6855Register fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6856 switch (VT.SimpleTy) {
6857 case MVT::v8i8: return fastEmit_ISD_UADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
6858 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
6859 case MVT::v4i16: return fastEmit_ISD_UADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
6860 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
6861 case MVT::v2i32: return fastEmit_ISD_UADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
6862 case MVT::v4i32: return fastEmit_ISD_UADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
6863 case MVT::v1i64: return fastEmit_ISD_UADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
6864 case MVT::v2i64: return fastEmit_ISD_UADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
6865 default: return Register();
6866 }
6867}
6868
6869// FastEmit functions for ISD::UDIV.
6870
6871Register fastEmit_ISD_UDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6872 if (RetVT.SimpleTy != MVT::i32)
6873 return Register();
6874 if ((Subtarget->hasDivideInThumbMode()) && (Subtarget->hasV8MBaselineOps()) && (Subtarget->isThumb())) {
6875 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UDIV, RC: &ARM::rGPRRegClass, Op0, Op1);
6876 }
6877 if ((Subtarget->hasDivideInARMMode()) && (!Subtarget->isThumb())) {
6878 return fastEmitInst_rr(MachineInstOpcode: ARM::UDIV, RC: &ARM::GPRRegClass, Op0, Op1);
6879 }
6880 return Register();
6881}
6882
6883Register fastEmit_ISD_UDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6884 switch (VT.SimpleTy) {
6885 case MVT::i32: return fastEmit_ISD_UDIV_MVT_i32_rr(RetVT, Op0, Op1);
6886 default: return Register();
6887 }
6888}
6889
6890// FastEmit functions for ISD::UMAX.
6891
6892Register fastEmit_ISD_UMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6893 if (RetVT.SimpleTy != MVT::v8i8)
6894 return Register();
6895 if ((Subtarget->hasNEON())) {
6896 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6897 }
6898 return Register();
6899}
6900
6901Register fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6902 if (RetVT.SimpleTy != MVT::v16i8)
6903 return Register();
6904 if ((Subtarget->hasMVEIntegerOps())) {
6905 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXu8, RC: &ARM::MQPRRegClass, Op0, Op1);
6906 }
6907 if ((Subtarget->hasNEON())) {
6908 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6909 }
6910 return Register();
6911}
6912
6913Register fastEmit_ISD_UMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6914 if (RetVT.SimpleTy != MVT::v4i16)
6915 return Register();
6916 if ((Subtarget->hasNEON())) {
6917 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6918 }
6919 return Register();
6920}
6921
6922Register fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6923 if (RetVT.SimpleTy != MVT::v8i16)
6924 return Register();
6925 if ((Subtarget->hasMVEIntegerOps())) {
6926 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXu16, RC: &ARM::MQPRRegClass, Op0, Op1);
6927 }
6928 if ((Subtarget->hasNEON())) {
6929 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6930 }
6931 return Register();
6932}
6933
6934Register fastEmit_ISD_UMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6935 if (RetVT.SimpleTy != MVT::v2i32)
6936 return Register();
6937 if ((Subtarget->hasNEON())) {
6938 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6939 }
6940 return Register();
6941}
6942
6943Register fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6944 if (RetVT.SimpleTy != MVT::v4i32)
6945 return Register();
6946 if ((Subtarget->hasMVEIntegerOps())) {
6947 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXu32, RC: &ARM::MQPRRegClass, Op0, Op1);
6948 }
6949 if ((Subtarget->hasNEON())) {
6950 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6951 }
6952 return Register();
6953}
6954
6955Register fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6956 switch (VT.SimpleTy) {
6957 case MVT::v8i8: return fastEmit_ISD_UMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
6958 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
6959 case MVT::v4i16: return fastEmit_ISD_UMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
6960 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
6961 case MVT::v2i32: return fastEmit_ISD_UMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
6962 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
6963 default: return Register();
6964 }
6965}
6966
6967// FastEmit functions for ISD::UMIN.
6968
6969Register fastEmit_ISD_UMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6970 if (RetVT.SimpleTy != MVT::v8i8)
6971 return Register();
6972 if ((Subtarget->hasNEON())) {
6973 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6974 }
6975 return Register();
6976}
6977
6978Register fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6979 if (RetVT.SimpleTy != MVT::v16i8)
6980 return Register();
6981 if ((Subtarget->hasMVEIntegerOps())) {
6982 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINu8, RC: &ARM::MQPRRegClass, Op0, Op1);
6983 }
6984 if ((Subtarget->hasNEON())) {
6985 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6986 }
6987 return Register();
6988}
6989
6990Register fastEmit_ISD_UMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6991 if (RetVT.SimpleTy != MVT::v4i16)
6992 return Register();
6993 if ((Subtarget->hasNEON())) {
6994 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6995 }
6996 return Register();
6997}
6998
6999Register fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7000 if (RetVT.SimpleTy != MVT::v8i16)
7001 return Register();
7002 if ((Subtarget->hasMVEIntegerOps())) {
7003 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINu16, RC: &ARM::MQPRRegClass, Op0, Op1);
7004 }
7005 if ((Subtarget->hasNEON())) {
7006 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
7007 }
7008 return Register();
7009}
7010
7011Register fastEmit_ISD_UMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7012 if (RetVT.SimpleTy != MVT::v2i32)
7013 return Register();
7014 if ((Subtarget->hasNEON())) {
7015 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
7016 }
7017 return Register();
7018}
7019
7020Register fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7021 if (RetVT.SimpleTy != MVT::v4i32)
7022 return Register();
7023 if ((Subtarget->hasMVEIntegerOps())) {
7024 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINu32, RC: &ARM::MQPRRegClass, Op0, Op1);
7025 }
7026 if ((Subtarget->hasNEON())) {
7027 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
7028 }
7029 return Register();
7030}
7031
7032Register fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7033 switch (VT.SimpleTy) {
7034 case MVT::v8i8: return fastEmit_ISD_UMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
7035 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
7036 case MVT::v4i16: return fastEmit_ISD_UMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
7037 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
7038 case MVT::v2i32: return fastEmit_ISD_UMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
7039 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
7040 default: return Register();
7041 }
7042}
7043
7044// FastEmit functions for ISD::USUBSAT.
7045
7046Register fastEmit_ISD_USUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7047 if (RetVT.SimpleTy != MVT::v8i8)
7048 return Register();
7049 if ((Subtarget->hasNEON())) {
7050 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
7051 }
7052 return Register();
7053}
7054
7055Register fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7056 if (RetVT.SimpleTy != MVT::v16i8)
7057 return Register();
7058 if ((Subtarget->hasMVEIntegerOps())) {
7059 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBu8, RC: &ARM::MQPRRegClass, Op0, Op1);
7060 }
7061 if ((Subtarget->hasNEON())) {
7062 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
7063 }
7064 return Register();
7065}
7066
7067Register fastEmit_ISD_USUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7068 if (RetVT.SimpleTy != MVT::v4i16)
7069 return Register();
7070 if ((Subtarget->hasNEON())) {
7071 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
7072 }
7073 return Register();
7074}
7075
7076Register fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7077 if (RetVT.SimpleTy != MVT::v8i16)
7078 return Register();
7079 if ((Subtarget->hasMVEIntegerOps())) {
7080 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBu16, RC: &ARM::MQPRRegClass, Op0, Op1);
7081 }
7082 if ((Subtarget->hasNEON())) {
7083 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
7084 }
7085 return Register();
7086}
7087
7088Register fastEmit_ISD_USUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7089 if (RetVT.SimpleTy != MVT::v2i32)
7090 return Register();
7091 if ((Subtarget->hasNEON())) {
7092 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
7093 }
7094 return Register();
7095}
7096
7097Register fastEmit_ISD_USUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7098 if (RetVT.SimpleTy != MVT::v4i32)
7099 return Register();
7100 if ((Subtarget->hasMVEIntegerOps())) {
7101 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBu32, RC: &ARM::MQPRRegClass, Op0, Op1);
7102 }
7103 if ((Subtarget->hasNEON())) {
7104 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
7105 }
7106 return Register();
7107}
7108
7109Register fastEmit_ISD_USUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7110 if (RetVT.SimpleTy != MVT::v1i64)
7111 return Register();
7112 if ((Subtarget->hasNEON())) {
7113 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
7114 }
7115 return Register();
7116}
7117
7118Register fastEmit_ISD_USUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7119 if (RetVT.SimpleTy != MVT::v2i64)
7120 return Register();
7121 if ((Subtarget->hasNEON())) {
7122 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
7123 }
7124 return Register();
7125}
7126
7127Register fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7128 switch (VT.SimpleTy) {
7129 case MVT::v8i8: return fastEmit_ISD_USUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
7130 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
7131 case MVT::v4i16: return fastEmit_ISD_USUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
7132 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
7133 case MVT::v2i32: return fastEmit_ISD_USUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
7134 case MVT::v4i32: return fastEmit_ISD_USUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
7135 case MVT::v1i64: return fastEmit_ISD_USUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
7136 case MVT::v2i64: return fastEmit_ISD_USUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
7137 default: return Register();
7138 }
7139}
7140
7141// FastEmit functions for ISD::XOR.
7142
7143Register fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
7144 if (RetVT.SimpleTy != MVT::i32)
7145 return Register();
7146 if ((Subtarget->isThumb2())) {
7147 return fastEmitInst_rr(MachineInstOpcode: ARM::t2EORrr, RC: &ARM::rGPRRegClass, Op0, Op1);
7148 }
7149 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
7150 return fastEmitInst_rr(MachineInstOpcode: ARM::tEOR, RC: &ARM::tGPRRegClass, Op0, Op1);
7151 }
7152 if ((!Subtarget->isThumb())) {
7153 return fastEmitInst_rr(MachineInstOpcode: ARM::EORrr, RC: &ARM::GPRRegClass, Op0, Op1);
7154 }
7155 return Register();
7156}
7157
7158Register fastEmit_ISD_XOR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7159 if (RetVT.SimpleTy != MVT::v8i8)
7160 return Register();
7161 if ((Subtarget->hasNEON())) {
7162 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
7163 }
7164 return Register();
7165}
7166
7167Register fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7168 if (RetVT.SimpleTy != MVT::v16i8)
7169 return Register();
7170 if ((Subtarget->hasMVEIntegerOps())) {
7171 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
7172 }
7173 if ((Subtarget->hasNEON())) {
7174 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
7175 }
7176 return Register();
7177}
7178
7179Register fastEmit_ISD_XOR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7180 if (RetVT.SimpleTy != MVT::v4i16)
7181 return Register();
7182 if ((Subtarget->hasNEON())) {
7183 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
7184 }
7185 return Register();
7186}
7187
7188Register fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7189 if (RetVT.SimpleTy != MVT::v8i16)
7190 return Register();
7191 if ((Subtarget->hasMVEIntegerOps())) {
7192 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
7193 }
7194 if ((Subtarget->hasNEON())) {
7195 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
7196 }
7197 return Register();
7198}
7199
7200Register fastEmit_ISD_XOR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7201 if (RetVT.SimpleTy != MVT::v2i32)
7202 return Register();
7203 if ((Subtarget->hasNEON())) {
7204 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
7205 }
7206 return Register();
7207}
7208
7209Register fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7210 if (RetVT.SimpleTy != MVT::v4i32)
7211 return Register();
7212 if ((Subtarget->hasMVEIntegerOps())) {
7213 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
7214 }
7215 if ((Subtarget->hasNEON())) {
7216 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
7217 }
7218 return Register();
7219}
7220
7221Register fastEmit_ISD_XOR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7222 if (RetVT.SimpleTy != MVT::v1i64)
7223 return Register();
7224 if ((Subtarget->hasNEON())) {
7225 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
7226 }
7227 return Register();
7228}
7229
7230Register fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7231 if (RetVT.SimpleTy != MVT::v2i64)
7232 return Register();
7233 if ((Subtarget->hasMVEIntegerOps())) {
7234 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
7235 }
7236 if ((Subtarget->hasNEON())) {
7237 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
7238 }
7239 return Register();
7240}
7241
7242Register fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7243 switch (VT.SimpleTy) {
7244 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
7245 case MVT::v8i8: return fastEmit_ISD_XOR_MVT_v8i8_rr(RetVT, Op0, Op1);
7246 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
7247 case MVT::v4i16: return fastEmit_ISD_XOR_MVT_v4i16_rr(RetVT, Op0, Op1);
7248 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
7249 case MVT::v2i32: return fastEmit_ISD_XOR_MVT_v2i32_rr(RetVT, Op0, Op1);
7250 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
7251 case MVT::v1i64: return fastEmit_ISD_XOR_MVT_v1i64_rr(RetVT, Op0, Op1);
7252 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
7253 default: return Register();
7254 }
7255}
7256
7257// Top-level FastEmit function.
7258
7259Register fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, Register Op1) override {
7260 switch (Opcode) {
7261 case ARMISD::CMP: return fastEmit_ARMISD_CMP_rr(VT, RetVT, Op0, Op1);
7262 case ARMISD::CMPFP: return fastEmit_ARMISD_CMPFP_rr(VT, RetVT, Op0, Op1);
7263 case ARMISD::CMPFPE: return fastEmit_ARMISD_CMPFPE_rr(VT, RetVT, Op0, Op1);
7264 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_rr(VT, RetVT, Op0, Op1);
7265 case ARMISD::EH_SJLJ_LONGJMP: return fastEmit_ARMISD_EH_SJLJ_LONGJMP_rr(VT, RetVT, Op0, Op1);
7266 case ARMISD::EH_SJLJ_SETJMP: return fastEmit_ARMISD_EH_SJLJ_SETJMP_rr(VT, RetVT, Op0, Op1);
7267 case ARMISD::QADD16b: return fastEmit_ARMISD_QADD16b_rr(VT, RetVT, Op0, Op1);
7268 case ARMISD::QADD8b: return fastEmit_ARMISD_QADD8b_rr(VT, RetVT, Op0, Op1);
7269 case ARMISD::QSUB16b: return fastEmit_ARMISD_QSUB16b_rr(VT, RetVT, Op0, Op1);
7270 case ARMISD::QSUB8b: return fastEmit_ARMISD_QSUB8b_rr(VT, RetVT, Op0, Op1);
7271 case ARMISD::SMULWB: return fastEmit_ARMISD_SMULWB_rr(VT, RetVT, Op0, Op1);
7272 case ARMISD::SMULWT: return fastEmit_ARMISD_SMULWT_rr(VT, RetVT, Op0, Op1);
7273 case ARMISD::UQADD16b: return fastEmit_ARMISD_UQADD16b_rr(VT, RetVT, Op0, Op1);
7274 case ARMISD::UQADD8b: return fastEmit_ARMISD_UQADD8b_rr(VT, RetVT, Op0, Op1);
7275 case ARMISD::UQSUB16b: return fastEmit_ARMISD_UQSUB16b_rr(VT, RetVT, Op0, Op1);
7276 case ARMISD::UQSUB8b: return fastEmit_ARMISD_UQSUB8b_rr(VT, RetVT, Op0, Op1);
7277 case ARMISD::VMLAVs: return fastEmit_ARMISD_VMLAVs_rr(VT, RetVT, Op0, Op1);
7278 case ARMISD::VMLAVu: return fastEmit_ARMISD_VMLAVu_rr(VT, RetVT, Op0, Op1);
7279 case ARMISD::VMOVDRR: return fastEmit_ARMISD_VMOVDRR_rr(VT, RetVT, Op0, Op1);
7280 case ARMISD::VMULLs: return fastEmit_ARMISD_VMULLs_rr(VT, RetVT, Op0, Op1);
7281 case ARMISD::VMULLu: return fastEmit_ARMISD_VMULLu_rr(VT, RetVT, Op0, Op1);
7282 case ARMISD::VQDMULH: return fastEmit_ARMISD_VQDMULH_rr(VT, RetVT, Op0, Op1);
7283 case ARMISD::VSHLs: return fastEmit_ARMISD_VSHLs_rr(VT, RetVT, Op0, Op1);
7284 case ARMISD::VSHLu: return fastEmit_ARMISD_VSHLu_rr(VT, RetVT, Op0, Op1);
7285 case ARMISD::VTBL1: return fastEmit_ARMISD_VTBL1_rr(VT, RetVT, Op0, Op1);
7286 case ARMISD::VTST: return fastEmit_ARMISD_VTST_rr(VT, RetVT, Op0, Op1);
7287 case ISD::ABDS: return fastEmit_ISD_ABDS_rr(VT, RetVT, Op0, Op1);
7288 case ISD::ABDU: return fastEmit_ISD_ABDU_rr(VT, RetVT, Op0, Op1);
7289 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
7290 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
7291 case ISD::AVGCEILS: return fastEmit_ISD_AVGCEILS_rr(VT, RetVT, Op0, Op1);
7292 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
7293 case ISD::AVGFLOORS: return fastEmit_ISD_AVGFLOORS_rr(VT, RetVT, Op0, Op1);
7294 case ISD::AVGFLOORU: return fastEmit_ISD_AVGFLOORU_rr(VT, RetVT, Op0, Op1);
7295 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
7296 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
7297 case ISD::FMAXIMUM: return fastEmit_ISD_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
7298 case ISD::FMAXNUM: return fastEmit_ISD_FMAXNUM_rr(VT, RetVT, Op0, Op1);
7299 case ISD::FMINIMUM: return fastEmit_ISD_FMINIMUM_rr(VT, RetVT, Op0, Op1);
7300 case ISD::FMINNUM: return fastEmit_ISD_FMINNUM_rr(VT, RetVT, Op0, Op1);
7301 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
7302 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
7303 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
7304 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
7305 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
7306 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
7307 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
7308 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
7309 case ISD::SDIV: return fastEmit_ISD_SDIV_rr(VT, RetVT, Op0, Op1);
7310 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
7311 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
7312 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
7313 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
7314 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
7315 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
7316 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1);
7317 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1);
7318 case ISD::STRICT_FMAXNUM: return fastEmit_ISD_STRICT_FMAXNUM_rr(VT, RetVT, Op0, Op1);
7319 case ISD::STRICT_FMINNUM: return fastEmit_ISD_STRICT_FMINNUM_rr(VT, RetVT, Op0, Op1);
7320 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1);
7321 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1);
7322 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
7323 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
7324 case ISD::UDIV: return fastEmit_ISD_UDIV_rr(VT, RetVT, Op0, Op1);
7325 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
7326 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
7327 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
7328 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
7329 default: return Register();
7330 }
7331}
7332
7333// FastEmit functions for ARMISD::PIC_ADD.
7334
7335Register fastEmit_ARMISD_PIC_ADD_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7336 if (RetVT.SimpleTy != MVT::i32)
7337 return Register();
7338 if ((Subtarget->isThumb())) {
7339 return fastEmitInst_ri(MachineInstOpcode: ARM::tPICADD, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7340 }
7341 if ((!Subtarget->isThumb())) {
7342 return fastEmitInst_ri(MachineInstOpcode: ARM::PICADD, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7343 }
7344 return Register();
7345}
7346
7347Register fastEmit_ARMISD_PIC_ADD_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7348 switch (VT.SimpleTy) {
7349 case MVT::i32: return fastEmit_ARMISD_PIC_ADD_MVT_i32_ri(RetVT, Op0, imm1);
7350 default: return Register();
7351 }
7352}
7353
7354// FastEmit functions for ARMISD::VDUPLANE.
7355
7356Register fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7357 if (RetVT.SimpleTy != MVT::v8i8)
7358 return Register();
7359 if ((Subtarget->hasNEON())) {
7360 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN8d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7361 }
7362 return Register();
7363}
7364
7365Register fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7366 if (RetVT.SimpleTy != MVT::v4i16)
7367 return Register();
7368 if ((Subtarget->hasNEON())) {
7369 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7370 }
7371 return Register();
7372}
7373
7374Register fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7375 if (RetVT.SimpleTy != MVT::v2i32)
7376 return Register();
7377 if ((Subtarget->hasNEON())) {
7378 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7379 }
7380 return Register();
7381}
7382
7383Register fastEmit_ARMISD_VDUPLANE_MVT_v4f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7384 if (RetVT.SimpleTy != MVT::v4f16)
7385 return Register();
7386 if ((Subtarget->hasNEON())) {
7387 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7388 }
7389 return Register();
7390}
7391
7392Register fastEmit_ARMISD_VDUPLANE_MVT_v4bf16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7393 if (RetVT.SimpleTy != MVT::v4bf16)
7394 return Register();
7395 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
7396 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7397 }
7398 return Register();
7399}
7400
7401Register fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v2f32_ri(Register Op0, uint64_t imm1) {
7402 if ((Subtarget->hasNEON())) {
7403 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7404 }
7405 return Register();
7406}
7407
7408Register fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v4f32_ri(Register Op0, uint64_t imm1) {
7409 if ((Subtarget->hasNEON())) {
7410 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7411 }
7412 return Register();
7413}
7414
7415Register fastEmit_ARMISD_VDUPLANE_MVT_v2f32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7416switch (RetVT.SimpleTy) {
7417 case MVT::v2f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v2f32_ri(Op0, imm1);
7418 case MVT::v4f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v4f32_ri(Op0, imm1);
7419 default: return Register();
7420}
7421}
7422
7423Register fastEmit_ARMISD_VDUPLANE_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7424 switch (VT.SimpleTy) {
7425 case MVT::v8i8: return fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri(RetVT, Op0, imm1);
7426 case MVT::v4i16: return fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri(RetVT, Op0, imm1);
7427 case MVT::v2i32: return fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri(RetVT, Op0, imm1);
7428 case MVT::v4f16: return fastEmit_ARMISD_VDUPLANE_MVT_v4f16_ri(RetVT, Op0, imm1);
7429 case MVT::v4bf16: return fastEmit_ARMISD_VDUPLANE_MVT_v4bf16_ri(RetVT, Op0, imm1);
7430 case MVT::v2f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_ri(RetVT, Op0, imm1);
7431 default: return Register();
7432 }
7433}
7434
7435// FastEmit functions for ARMISD::VGETLANEs.
7436
7437Register fastEmit_ARMISD_VGETLANEs_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7438 if (RetVT.SimpleTy != MVT::i32)
7439 return Register();
7440 if ((Subtarget->hasNEON())) {
7441 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNs8, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7442 }
7443 return Register();
7444}
7445
7446Register fastEmit_ARMISD_VGETLANEs_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7447 if (RetVT.SimpleTy != MVT::i32)
7448 return Register();
7449 if ((Subtarget->hasMVEIntegerOps())) {
7450 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_s8, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7451 }
7452 return Register();
7453}
7454
7455Register fastEmit_ARMISD_VGETLANEs_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7456 if (RetVT.SimpleTy != MVT::i32)
7457 return Register();
7458 if ((Subtarget->hasNEON())) {
7459 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNs16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7460 }
7461 return Register();
7462}
7463
7464Register fastEmit_ARMISD_VGETLANEs_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7465 if (RetVT.SimpleTy != MVT::i32)
7466 return Register();
7467 if ((Subtarget->hasMVEIntegerOps())) {
7468 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_s16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7469 }
7470 return Register();
7471}
7472
7473Register fastEmit_ARMISD_VGETLANEs_MVT_v8f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7474 if (RetVT.SimpleTy != MVT::i32)
7475 return Register();
7476 if ((Subtarget->hasMVEIntegerOps())) {
7477 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_s16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7478 }
7479 return Register();
7480}
7481
7482Register fastEmit_ARMISD_VGETLANEs_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7483 switch (VT.SimpleTy) {
7484 case MVT::v8i8: return fastEmit_ARMISD_VGETLANEs_MVT_v8i8_ri(RetVT, Op0, imm1);
7485 case MVT::v16i8: return fastEmit_ARMISD_VGETLANEs_MVT_v16i8_ri(RetVT, Op0, imm1);
7486 case MVT::v4i16: return fastEmit_ARMISD_VGETLANEs_MVT_v4i16_ri(RetVT, Op0, imm1);
7487 case MVT::v8i16: return fastEmit_ARMISD_VGETLANEs_MVT_v8i16_ri(RetVT, Op0, imm1);
7488 case MVT::v8f16: return fastEmit_ARMISD_VGETLANEs_MVT_v8f16_ri(RetVT, Op0, imm1);
7489 default: return Register();
7490 }
7491}
7492
7493// FastEmit functions for ARMISD::VGETLANEu.
7494
7495Register fastEmit_ARMISD_VGETLANEu_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7496 if (RetVT.SimpleTy != MVT::i32)
7497 return Register();
7498 if ((Subtarget->hasNEON())) {
7499 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu8, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7500 }
7501 return Register();
7502}
7503
7504Register fastEmit_ARMISD_VGETLANEu_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7505 if (RetVT.SimpleTy != MVT::i32)
7506 return Register();
7507 if ((Subtarget->hasMVEIntegerOps())) {
7508 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_u8, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7509 }
7510 return Register();
7511}
7512
7513Register fastEmit_ARMISD_VGETLANEu_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7514 if (RetVT.SimpleTy != MVT::i32)
7515 return Register();
7516 if ((Subtarget->hasNEON())) {
7517 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7518 }
7519 return Register();
7520}
7521
7522Register fastEmit_ARMISD_VGETLANEu_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7523 if (RetVT.SimpleTy != MVT::i32)
7524 return Register();
7525 if ((Subtarget->hasMVEIntegerOps())) {
7526 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_u16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7527 }
7528 return Register();
7529}
7530
7531Register fastEmit_ARMISD_VGETLANEu_MVT_v4f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7532 if (RetVT.SimpleTy != MVT::i32)
7533 return Register();
7534 if ((Subtarget->hasNEON())) {
7535 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7536 }
7537 return Register();
7538}
7539
7540Register fastEmit_ARMISD_VGETLANEu_MVT_v8f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7541 if (RetVT.SimpleTy != MVT::i32)
7542 return Register();
7543 if ((Subtarget->hasMVEIntegerOps())) {
7544 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_u16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7545 }
7546 return Register();
7547}
7548
7549Register fastEmit_ARMISD_VGETLANEu_MVT_v4bf16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7550 if (RetVT.SimpleTy != MVT::i32)
7551 return Register();
7552 if ((Subtarget->hasNEON())) {
7553 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7554 }
7555 return Register();
7556}
7557
7558Register fastEmit_ARMISD_VGETLANEu_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7559 switch (VT.SimpleTy) {
7560 case MVT::v8i8: return fastEmit_ARMISD_VGETLANEu_MVT_v8i8_ri(RetVT, Op0, imm1);
7561 case MVT::v16i8: return fastEmit_ARMISD_VGETLANEu_MVT_v16i8_ri(RetVT, Op0, imm1);
7562 case MVT::v4i16: return fastEmit_ARMISD_VGETLANEu_MVT_v4i16_ri(RetVT, Op0, imm1);
7563 case MVT::v8i16: return fastEmit_ARMISD_VGETLANEu_MVT_v8i16_ri(RetVT, Op0, imm1);
7564 case MVT::v4f16: return fastEmit_ARMISD_VGETLANEu_MVT_v4f16_ri(RetVT, Op0, imm1);
7565 case MVT::v8f16: return fastEmit_ARMISD_VGETLANEu_MVT_v8f16_ri(RetVT, Op0, imm1);
7566 case MVT::v4bf16: return fastEmit_ARMISD_VGETLANEu_MVT_v4bf16_ri(RetVT, Op0, imm1);
7567 default: return Register();
7568 }
7569}
7570
7571// FastEmit functions for ARMISD::VQSHLsIMM.
7572
7573Register fastEmit_ARMISD_VQSHLsIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7574 if (RetVT.SimpleTy != MVT::v8i8)
7575 return Register();
7576 if ((Subtarget->hasNEON())) {
7577 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7578 }
7579 return Register();
7580}
7581
7582Register fastEmit_ARMISD_VQSHLsIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7583 if (RetVT.SimpleTy != MVT::v16i8)
7584 return Register();
7585 if ((Subtarget->hasNEON())) {
7586 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7587 }
7588 return Register();
7589}
7590
7591Register fastEmit_ARMISD_VQSHLsIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7592 if (RetVT.SimpleTy != MVT::v4i16)
7593 return Register();
7594 if ((Subtarget->hasNEON())) {
7595 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7596 }
7597 return Register();
7598}
7599
7600Register fastEmit_ARMISD_VQSHLsIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7601 if (RetVT.SimpleTy != MVT::v8i16)
7602 return Register();
7603 if ((Subtarget->hasNEON())) {
7604 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7605 }
7606 return Register();
7607}
7608
7609Register fastEmit_ARMISD_VQSHLsIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7610 if (RetVT.SimpleTy != MVT::v2i32)
7611 return Register();
7612 if ((Subtarget->hasNEON())) {
7613 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7614 }
7615 return Register();
7616}
7617
7618Register fastEmit_ARMISD_VQSHLsIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7619 if (RetVT.SimpleTy != MVT::v4i32)
7620 return Register();
7621 if ((Subtarget->hasNEON())) {
7622 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7623 }
7624 return Register();
7625}
7626
7627Register fastEmit_ARMISD_VQSHLsIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7628 if (RetVT.SimpleTy != MVT::v1i64)
7629 return Register();
7630 if ((Subtarget->hasNEON())) {
7631 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7632 }
7633 return Register();
7634}
7635
7636Register fastEmit_ARMISD_VQSHLsIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7637 if (RetVT.SimpleTy != MVT::v2i64)
7638 return Register();
7639 if ((Subtarget->hasNEON())) {
7640 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7641 }
7642 return Register();
7643}
7644
7645Register fastEmit_ARMISD_VQSHLsIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7646 switch (VT.SimpleTy) {
7647 case MVT::v8i8: return fastEmit_ARMISD_VQSHLsIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7648 case MVT::v16i8: return fastEmit_ARMISD_VQSHLsIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7649 case MVT::v4i16: return fastEmit_ARMISD_VQSHLsIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7650 case MVT::v8i16: return fastEmit_ARMISD_VQSHLsIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7651 case MVT::v2i32: return fastEmit_ARMISD_VQSHLsIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7652 case MVT::v4i32: return fastEmit_ARMISD_VQSHLsIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7653 case MVT::v1i64: return fastEmit_ARMISD_VQSHLsIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7654 case MVT::v2i64: return fastEmit_ARMISD_VQSHLsIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7655 default: return Register();
7656 }
7657}
7658
7659// FastEmit functions for ARMISD::VQSHLsuIMM.
7660
7661Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7662 if (RetVT.SimpleTy != MVT::v8i8)
7663 return Register();
7664 if ((Subtarget->hasNEON())) {
7665 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7666 }
7667 return Register();
7668}
7669
7670Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7671 if (RetVT.SimpleTy != MVT::v16i8)
7672 return Register();
7673 if ((Subtarget->hasNEON())) {
7674 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7675 }
7676 return Register();
7677}
7678
7679Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7680 if (RetVT.SimpleTy != MVT::v4i16)
7681 return Register();
7682 if ((Subtarget->hasNEON())) {
7683 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7684 }
7685 return Register();
7686}
7687
7688Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7689 if (RetVT.SimpleTy != MVT::v8i16)
7690 return Register();
7691 if ((Subtarget->hasNEON())) {
7692 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7693 }
7694 return Register();
7695}
7696
7697Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7698 if (RetVT.SimpleTy != MVT::v2i32)
7699 return Register();
7700 if ((Subtarget->hasNEON())) {
7701 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7702 }
7703 return Register();
7704}
7705
7706Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7707 if (RetVT.SimpleTy != MVT::v4i32)
7708 return Register();
7709 if ((Subtarget->hasNEON())) {
7710 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7711 }
7712 return Register();
7713}
7714
7715Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7716 if (RetVT.SimpleTy != MVT::v1i64)
7717 return Register();
7718 if ((Subtarget->hasNEON())) {
7719 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7720 }
7721 return Register();
7722}
7723
7724Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7725 if (RetVT.SimpleTy != MVT::v2i64)
7726 return Register();
7727 if ((Subtarget->hasNEON())) {
7728 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7729 }
7730 return Register();
7731}
7732
7733Register fastEmit_ARMISD_VQSHLsuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7734 switch (VT.SimpleTy) {
7735 case MVT::v8i8: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7736 case MVT::v16i8: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7737 case MVT::v4i16: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7738 case MVT::v8i16: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7739 case MVT::v2i32: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7740 case MVT::v4i32: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7741 case MVT::v1i64: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7742 case MVT::v2i64: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7743 default: return Register();
7744 }
7745}
7746
7747// FastEmit functions for ARMISD::VQSHLuIMM.
7748
7749Register fastEmit_ARMISD_VQSHLuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7750 if (RetVT.SimpleTy != MVT::v8i8)
7751 return Register();
7752 if ((Subtarget->hasNEON())) {
7753 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7754 }
7755 return Register();
7756}
7757
7758Register fastEmit_ARMISD_VQSHLuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7759 if (RetVT.SimpleTy != MVT::v16i8)
7760 return Register();
7761 if ((Subtarget->hasNEON())) {
7762 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7763 }
7764 return Register();
7765}
7766
7767Register fastEmit_ARMISD_VQSHLuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7768 if (RetVT.SimpleTy != MVT::v4i16)
7769 return Register();
7770 if ((Subtarget->hasNEON())) {
7771 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7772 }
7773 return Register();
7774}
7775
7776Register fastEmit_ARMISD_VQSHLuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7777 if (RetVT.SimpleTy != MVT::v8i16)
7778 return Register();
7779 if ((Subtarget->hasNEON())) {
7780 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7781 }
7782 return Register();
7783}
7784
7785Register fastEmit_ARMISD_VQSHLuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7786 if (RetVT.SimpleTy != MVT::v2i32)
7787 return Register();
7788 if ((Subtarget->hasNEON())) {
7789 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7790 }
7791 return Register();
7792}
7793
7794Register fastEmit_ARMISD_VQSHLuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7795 if (RetVT.SimpleTy != MVT::v4i32)
7796 return Register();
7797 if ((Subtarget->hasNEON())) {
7798 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7799 }
7800 return Register();
7801}
7802
7803Register fastEmit_ARMISD_VQSHLuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7804 if (RetVT.SimpleTy != MVT::v1i64)
7805 return Register();
7806 if ((Subtarget->hasNEON())) {
7807 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7808 }
7809 return Register();
7810}
7811
7812Register fastEmit_ARMISD_VQSHLuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7813 if (RetVT.SimpleTy != MVT::v2i64)
7814 return Register();
7815 if ((Subtarget->hasNEON())) {
7816 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7817 }
7818 return Register();
7819}
7820
7821Register fastEmit_ARMISD_VQSHLuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7822 switch (VT.SimpleTy) {
7823 case MVT::v8i8: return fastEmit_ARMISD_VQSHLuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7824 case MVT::v16i8: return fastEmit_ARMISD_VQSHLuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7825 case MVT::v4i16: return fastEmit_ARMISD_VQSHLuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7826 case MVT::v8i16: return fastEmit_ARMISD_VQSHLuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7827 case MVT::v2i32: return fastEmit_ARMISD_VQSHLuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7828 case MVT::v4i32: return fastEmit_ARMISD_VQSHLuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7829 case MVT::v1i64: return fastEmit_ARMISD_VQSHLuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7830 case MVT::v2i64: return fastEmit_ARMISD_VQSHLuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7831 default: return Register();
7832 }
7833}
7834
7835// FastEmit functions for ARMISD::VRSHRsIMM.
7836
7837Register fastEmit_ARMISD_VRSHRsIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7838 if (RetVT.SimpleTy != MVT::v8i8)
7839 return Register();
7840 if ((Subtarget->hasNEON())) {
7841 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7842 }
7843 return Register();
7844}
7845
7846Register fastEmit_ARMISD_VRSHRsIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7847 if (RetVT.SimpleTy != MVT::v16i8)
7848 return Register();
7849 if ((Subtarget->hasNEON())) {
7850 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7851 }
7852 return Register();
7853}
7854
7855Register fastEmit_ARMISD_VRSHRsIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7856 if (RetVT.SimpleTy != MVT::v4i16)
7857 return Register();
7858 if ((Subtarget->hasNEON())) {
7859 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7860 }
7861 return Register();
7862}
7863
7864Register fastEmit_ARMISD_VRSHRsIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7865 if (RetVT.SimpleTy != MVT::v8i16)
7866 return Register();
7867 if ((Subtarget->hasNEON())) {
7868 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7869 }
7870 return Register();
7871}
7872
7873Register fastEmit_ARMISD_VRSHRsIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7874 if (RetVT.SimpleTy != MVT::v2i32)
7875 return Register();
7876 if ((Subtarget->hasNEON())) {
7877 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7878 }
7879 return Register();
7880}
7881
7882Register fastEmit_ARMISD_VRSHRsIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7883 if (RetVT.SimpleTy != MVT::v4i32)
7884 return Register();
7885 if ((Subtarget->hasNEON())) {
7886 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7887 }
7888 return Register();
7889}
7890
7891Register fastEmit_ARMISD_VRSHRsIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7892 if (RetVT.SimpleTy != MVT::v1i64)
7893 return Register();
7894 if ((Subtarget->hasNEON())) {
7895 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7896 }
7897 return Register();
7898}
7899
7900Register fastEmit_ARMISD_VRSHRsIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7901 if (RetVT.SimpleTy != MVT::v2i64)
7902 return Register();
7903 if ((Subtarget->hasNEON())) {
7904 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7905 }
7906 return Register();
7907}
7908
7909Register fastEmit_ARMISD_VRSHRsIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7910 switch (VT.SimpleTy) {
7911 case MVT::v8i8: return fastEmit_ARMISD_VRSHRsIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7912 case MVT::v16i8: return fastEmit_ARMISD_VRSHRsIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7913 case MVT::v4i16: return fastEmit_ARMISD_VRSHRsIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7914 case MVT::v8i16: return fastEmit_ARMISD_VRSHRsIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7915 case MVT::v2i32: return fastEmit_ARMISD_VRSHRsIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7916 case MVT::v4i32: return fastEmit_ARMISD_VRSHRsIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7917 case MVT::v1i64: return fastEmit_ARMISD_VRSHRsIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7918 case MVT::v2i64: return fastEmit_ARMISD_VRSHRsIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7919 default: return Register();
7920 }
7921}
7922
7923// FastEmit functions for ARMISD::VRSHRuIMM.
7924
7925Register fastEmit_ARMISD_VRSHRuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7926 if (RetVT.SimpleTy != MVT::v8i8)
7927 return Register();
7928 if ((Subtarget->hasNEON())) {
7929 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7930 }
7931 return Register();
7932}
7933
7934Register fastEmit_ARMISD_VRSHRuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7935 if (RetVT.SimpleTy != MVT::v16i8)
7936 return Register();
7937 if ((Subtarget->hasNEON())) {
7938 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7939 }
7940 return Register();
7941}
7942
7943Register fastEmit_ARMISD_VRSHRuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7944 if (RetVT.SimpleTy != MVT::v4i16)
7945 return Register();
7946 if ((Subtarget->hasNEON())) {
7947 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7948 }
7949 return Register();
7950}
7951
7952Register fastEmit_ARMISD_VRSHRuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7953 if (RetVT.SimpleTy != MVT::v8i16)
7954 return Register();
7955 if ((Subtarget->hasNEON())) {
7956 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7957 }
7958 return Register();
7959}
7960
7961Register fastEmit_ARMISD_VRSHRuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7962 if (RetVT.SimpleTy != MVT::v2i32)
7963 return Register();
7964 if ((Subtarget->hasNEON())) {
7965 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7966 }
7967 return Register();
7968}
7969
7970Register fastEmit_ARMISD_VRSHRuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7971 if (RetVT.SimpleTy != MVT::v4i32)
7972 return Register();
7973 if ((Subtarget->hasNEON())) {
7974 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7975 }
7976 return Register();
7977}
7978
7979Register fastEmit_ARMISD_VRSHRuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7980 if (RetVT.SimpleTy != MVT::v1i64)
7981 return Register();
7982 if ((Subtarget->hasNEON())) {
7983 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7984 }
7985 return Register();
7986}
7987
7988Register fastEmit_ARMISD_VRSHRuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7989 if (RetVT.SimpleTy != MVT::v2i64)
7990 return Register();
7991 if ((Subtarget->hasNEON())) {
7992 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7993 }
7994 return Register();
7995}
7996
7997Register fastEmit_ARMISD_VRSHRuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7998 switch (VT.SimpleTy) {
7999 case MVT::v8i8: return fastEmit_ARMISD_VRSHRuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8000 case MVT::v16i8: return fastEmit_ARMISD_VRSHRuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8001 case MVT::v4i16: return fastEmit_ARMISD_VRSHRuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8002 case MVT::v8i16: return fastEmit_ARMISD_VRSHRuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8003 case MVT::v2i32: return fastEmit_ARMISD_VRSHRuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8004 case MVT::v4i32: return fastEmit_ARMISD_VRSHRuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8005 case MVT::v1i64: return fastEmit_ARMISD_VRSHRuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8006 case MVT::v2i64: return fastEmit_ARMISD_VRSHRuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8007 default: return Register();
8008 }
8009}
8010
8011// FastEmit functions for ARMISD::VSHLIMM.
8012
8013Register fastEmit_ARMISD_VSHLIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8014 if (RetVT.SimpleTy != MVT::v8i8)
8015 return Register();
8016 if ((Subtarget->hasNEON())) {
8017 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8018 }
8019 return Register();
8020}
8021
8022Register fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8023 if (RetVT.SimpleTy != MVT::v16i8)
8024 return Register();
8025 if ((Subtarget->hasNEON())) {
8026 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8027 }
8028 return Register();
8029}
8030
8031Register fastEmit_ARMISD_VSHLIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8032 if (RetVT.SimpleTy != MVT::v4i16)
8033 return Register();
8034 if ((Subtarget->hasNEON())) {
8035 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8036 }
8037 return Register();
8038}
8039
8040Register fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8041 if (RetVT.SimpleTy != MVT::v8i16)
8042 return Register();
8043 if ((Subtarget->hasNEON())) {
8044 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8045 }
8046 return Register();
8047}
8048
8049Register fastEmit_ARMISD_VSHLIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8050 if (RetVT.SimpleTy != MVT::v2i32)
8051 return Register();
8052 if ((Subtarget->hasNEON())) {
8053 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8054 }
8055 return Register();
8056}
8057
8058Register fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8059 if (RetVT.SimpleTy != MVT::v4i32)
8060 return Register();
8061 if ((Subtarget->hasNEON())) {
8062 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8063 }
8064 return Register();
8065}
8066
8067Register fastEmit_ARMISD_VSHLIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8068 if (RetVT.SimpleTy != MVT::v1i64)
8069 return Register();
8070 if ((Subtarget->hasNEON())) {
8071 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8072 }
8073 return Register();
8074}
8075
8076Register fastEmit_ARMISD_VSHLIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8077 if (RetVT.SimpleTy != MVT::v2i64)
8078 return Register();
8079 if ((Subtarget->hasNEON())) {
8080 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8081 }
8082 return Register();
8083}
8084
8085Register fastEmit_ARMISD_VSHLIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8086 switch (VT.SimpleTy) {
8087 case MVT::v8i8: return fastEmit_ARMISD_VSHLIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8088 case MVT::v16i8: return fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8089 case MVT::v4i16: return fastEmit_ARMISD_VSHLIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8090 case MVT::v8i16: return fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8091 case MVT::v2i32: return fastEmit_ARMISD_VSHLIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8092 case MVT::v4i32: return fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8093 case MVT::v1i64: return fastEmit_ARMISD_VSHLIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8094 case MVT::v2i64: return fastEmit_ARMISD_VSHLIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8095 default: return Register();
8096 }
8097}
8098
8099// FastEmit functions for ARMISD::VSHRsIMM.
8100
8101Register fastEmit_ARMISD_VSHRsIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8102 if (RetVT.SimpleTy != MVT::v8i8)
8103 return Register();
8104 if ((Subtarget->hasNEON())) {
8105 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8106 }
8107 return Register();
8108}
8109
8110Register fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8111 if (RetVT.SimpleTy != MVT::v16i8)
8112 return Register();
8113 if ((Subtarget->hasNEON())) {
8114 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8115 }
8116 return Register();
8117}
8118
8119Register fastEmit_ARMISD_VSHRsIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8120 if (RetVT.SimpleTy != MVT::v4i16)
8121 return Register();
8122 if ((Subtarget->hasNEON())) {
8123 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8124 }
8125 return Register();
8126}
8127
8128Register fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8129 if (RetVT.SimpleTy != MVT::v8i16)
8130 return Register();
8131 if ((Subtarget->hasNEON())) {
8132 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8133 }
8134 return Register();
8135}
8136
8137Register fastEmit_ARMISD_VSHRsIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8138 if (RetVT.SimpleTy != MVT::v2i32)
8139 return Register();
8140 if ((Subtarget->hasNEON())) {
8141 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8142 }
8143 return Register();
8144}
8145
8146Register fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8147 if (RetVT.SimpleTy != MVT::v4i32)
8148 return Register();
8149 if ((Subtarget->hasNEON())) {
8150 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8151 }
8152 return Register();
8153}
8154
8155Register fastEmit_ARMISD_VSHRsIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8156 if (RetVT.SimpleTy != MVT::v1i64)
8157 return Register();
8158 if ((Subtarget->hasNEON())) {
8159 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8160 }
8161 return Register();
8162}
8163
8164Register fastEmit_ARMISD_VSHRsIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8165 if (RetVT.SimpleTy != MVT::v2i64)
8166 return Register();
8167 if ((Subtarget->hasNEON())) {
8168 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8169 }
8170 return Register();
8171}
8172
8173Register fastEmit_ARMISD_VSHRsIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8174 switch (VT.SimpleTy) {
8175 case MVT::v8i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8176 case MVT::v16i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8177 case MVT::v4i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8178 case MVT::v8i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8179 case MVT::v2i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8180 case MVT::v4i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8181 case MVT::v1i64: return fastEmit_ARMISD_VSHRsIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8182 case MVT::v2i64: return fastEmit_ARMISD_VSHRsIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8183 default: return Register();
8184 }
8185}
8186
8187// FastEmit functions for ARMISD::VSHRuIMM.
8188
8189Register fastEmit_ARMISD_VSHRuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8190 if (RetVT.SimpleTy != MVT::v8i8)
8191 return Register();
8192 if ((Subtarget->hasNEON())) {
8193 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8194 }
8195 return Register();
8196}
8197
8198Register fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8199 if (RetVT.SimpleTy != MVT::v16i8)
8200 return Register();
8201 if ((Subtarget->hasNEON())) {
8202 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8203 }
8204 return Register();
8205}
8206
8207Register fastEmit_ARMISD_VSHRuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8208 if (RetVT.SimpleTy != MVT::v4i16)
8209 return Register();
8210 if ((Subtarget->hasNEON())) {
8211 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8212 }
8213 return Register();
8214}
8215
8216Register fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8217 if (RetVT.SimpleTy != MVT::v8i16)
8218 return Register();
8219 if ((Subtarget->hasNEON())) {
8220 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8221 }
8222 return Register();
8223}
8224
8225Register fastEmit_ARMISD_VSHRuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8226 if (RetVT.SimpleTy != MVT::v2i32)
8227 return Register();
8228 if ((Subtarget->hasNEON())) {
8229 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8230 }
8231 return Register();
8232}
8233
8234Register fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8235 if (RetVT.SimpleTy != MVT::v4i32)
8236 return Register();
8237 if ((Subtarget->hasNEON())) {
8238 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8239 }
8240 return Register();
8241}
8242
8243Register fastEmit_ARMISD_VSHRuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8244 if (RetVT.SimpleTy != MVT::v1i64)
8245 return Register();
8246 if ((Subtarget->hasNEON())) {
8247 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8248 }
8249 return Register();
8250}
8251
8252Register fastEmit_ARMISD_VSHRuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8253 if (RetVT.SimpleTy != MVT::v2i64)
8254 return Register();
8255 if ((Subtarget->hasNEON())) {
8256 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8257 }
8258 return Register();
8259}
8260
8261Register fastEmit_ARMISD_VSHRuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8262 switch (VT.SimpleTy) {
8263 case MVT::v8i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8264 case MVT::v16i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8265 case MVT::v4i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8266 case MVT::v8i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8267 case MVT::v2i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8268 case MVT::v4i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8269 case MVT::v1i64: return fastEmit_ARMISD_VSHRuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8270 case MVT::v2i64: return fastEmit_ARMISD_VSHRuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8271 default: return Register();
8272 }
8273}
8274
8275// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
8276
8277Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8278 if (RetVT.SimpleTy != MVT::i32)
8279 return Register();
8280 if ((Subtarget->hasFPRegs()) && (!Subtarget->hasSlowVGETLNi32())) {
8281 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNi32, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8282 }
8283 return Register();
8284}
8285
8286Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8287 switch (VT.SimpleTy) {
8288 case MVT::v2i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i32_ri(RetVT, Op0, imm1);
8289 default: return Register();
8290 }
8291}
8292
8293// FastEmit functions for ISD::SHL.
8294
8295Register fastEmit_ISD_SHL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8296 if (RetVT.SimpleTy != MVT::i32)
8297 return Register();
8298 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
8299 return fastEmitInst_ri(MachineInstOpcode: ARM::tLSLri, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
8300 }
8301 return Register();
8302}
8303
8304Register fastEmit_ISD_SHL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8305 switch (VT.SimpleTy) {
8306 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri(RetVT, Op0, imm1);
8307 default: return Register();
8308 }
8309}
8310
8311// Top-level FastEmit function.
8312
8313Register fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) override {
8314 if (VT == MVT::i32 && Predicate_mod_imm(Imm: imm1))
8315 if (Register Reg = fastEmit_ri_Predicate_mod_imm(VT, RetVT, Opcode, Op0, imm1))
8316 return Reg;
8317
8318 if (VT == MVT::i32 && Predicate_imm0_7(Imm: imm1))
8319 if (Register Reg = fastEmit_ri_Predicate_imm0_7(VT, RetVT, Opcode, Op0, imm1))
8320 return Reg;
8321
8322 if (VT == MVT::i32 && Predicate_imm0_255_expr(Imm: imm1))
8323 if (Register Reg = fastEmit_ri_Predicate_imm0_255_expr(VT, RetVT, Opcode, Op0, imm1))
8324 return Reg;
8325
8326 if (VT == MVT::i32 && Predicate_imm0_255(Imm: imm1))
8327 if (Register Reg = fastEmit_ri_Predicate_imm0_255(VT, RetVT, Opcode, Op0, imm1))
8328 return Reg;
8329
8330 if (VT == MVT::i32 && Predicate_t2_so_imm(Imm: imm1))
8331 if (Register Reg = fastEmit_ri_Predicate_t2_so_imm(VT, RetVT, Opcode, Op0, imm1))
8332 return Reg;
8333
8334 if (VT == MVT::i32 && Predicate_imm0_4095(Imm: imm1))
8335 if (Register Reg = fastEmit_ri_Predicate_imm0_4095(VT, RetVT, Opcode, Op0, imm1))
8336 return Reg;
8337
8338 if (VT == MVT::i32 && Predicate_imm1_31(Imm: imm1))
8339 if (Register Reg = fastEmit_ri_Predicate_imm1_31(VT, RetVT, Opcode, Op0, imm1))
8340 return Reg;
8341
8342 if (VT == MVT::i32 && Predicate_shr_imm8(Imm: imm1))
8343 if (Register Reg = fastEmit_ri_Predicate_shr_imm8(VT, RetVT, Opcode, Op0, imm1))
8344 return Reg;
8345
8346 if (VT == MVT::i32 && Predicate_shr_imm16(Imm: imm1))
8347 if (Register Reg = fastEmit_ri_Predicate_shr_imm16(VT, RetVT, Opcode, Op0, imm1))
8348 return Reg;
8349
8350 if (VT == MVT::i32 && Predicate_shr_imm32(Imm: imm1))
8351 if (Register Reg = fastEmit_ri_Predicate_shr_imm32(VT, RetVT, Opcode, Op0, imm1))
8352 return Reg;
8353
8354 if (VT == MVT::i32 && Predicate_VectorIndex32(Imm: imm1))
8355 if (Register Reg = fastEmit_ri_Predicate_VectorIndex32(VT, RetVT, Opcode, Op0, imm1))
8356 return Reg;
8357
8358 if (VT == MVT::i32 && Predicate_imm0_31(Imm: imm1))
8359 if (Register Reg = fastEmit_ri_Predicate_imm0_31(VT, RetVT, Opcode, Op0, imm1))
8360 return Reg;
8361
8362 if (VT == MVT::i32 && Predicate_imm0_15(Imm: imm1))
8363 if (Register Reg = fastEmit_ri_Predicate_imm0_15(VT, RetVT, Opcode, Op0, imm1))
8364 return Reg;
8365
8366 switch (Opcode) {
8367 case ARMISD::PIC_ADD: return fastEmit_ARMISD_PIC_ADD_ri(VT, RetVT, Op0, imm1);
8368 case ARMISD::VDUPLANE: return fastEmit_ARMISD_VDUPLANE_ri(VT, RetVT, Op0, imm1);
8369 case ARMISD::VGETLANEs: return fastEmit_ARMISD_VGETLANEs_ri(VT, RetVT, Op0, imm1);
8370 case ARMISD::VGETLANEu: return fastEmit_ARMISD_VGETLANEu_ri(VT, RetVT, Op0, imm1);
8371 case ARMISD::VQSHLsIMM: return fastEmit_ARMISD_VQSHLsIMM_ri(VT, RetVT, Op0, imm1);
8372 case ARMISD::VQSHLsuIMM: return fastEmit_ARMISD_VQSHLsuIMM_ri(VT, RetVT, Op0, imm1);
8373 case ARMISD::VQSHLuIMM: return fastEmit_ARMISD_VQSHLuIMM_ri(VT, RetVT, Op0, imm1);
8374 case ARMISD::VRSHRsIMM: return fastEmit_ARMISD_VRSHRsIMM_ri(VT, RetVT, Op0, imm1);
8375 case ARMISD::VRSHRuIMM: return fastEmit_ARMISD_VRSHRuIMM_ri(VT, RetVT, Op0, imm1);
8376 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri(VT, RetVT, Op0, imm1);
8377 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri(VT, RetVT, Op0, imm1);
8378 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri(VT, RetVT, Op0, imm1);
8379 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(VT, RetVT, Op0, imm1);
8380 case ISD::SHL: return fastEmit_ISD_SHL_ri(VT, RetVT, Op0, imm1);
8381 default: return Register();
8382 }
8383}
8384
8385// FastEmit functions for ARMISD::CMN.
8386
8387Register fastEmit_ARMISD_CMN_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8388 if (RetVT.SimpleTy != MVT::i32)
8389 return Register();
8390 if ((!Subtarget->isThumb())) {
8391 return fastEmitInst_ri(MachineInstOpcode: ARM::CMNri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8392 }
8393 return Register();
8394}
8395
8396Register fastEmit_ARMISD_CMN_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8397 switch (VT.SimpleTy) {
8398 case MVT::i32: return fastEmit_ARMISD_CMN_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8399 default: return Register();
8400 }
8401}
8402
8403// FastEmit functions for ARMISD::CMP.
8404
8405Register fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8406 if (RetVT.SimpleTy != MVT::i32)
8407 return Register();
8408 if ((!Subtarget->isThumb())) {
8409 return fastEmitInst_ri(MachineInstOpcode: ARM::CMPri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8410 }
8411 return Register();
8412}
8413
8414Register fastEmit_ARMISD_CMP_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8415 switch (VT.SimpleTy) {
8416 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8417 default: return Register();
8418 }
8419}
8420
8421// FastEmit functions for ARMISD::CMPZ.
8422
8423Register fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8424 if (RetVT.SimpleTy != MVT::i32)
8425 return Register();
8426 if ((!Subtarget->isThumb())) {
8427 return fastEmitInst_ri(MachineInstOpcode: ARM::CMPri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8428 }
8429 return Register();
8430}
8431
8432Register fastEmit_ARMISD_CMPZ_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8433 switch (VT.SimpleTy) {
8434 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8435 default: return Register();
8436 }
8437}
8438
8439// FastEmit functions for ISD::ADD.
8440
8441Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8442 if (RetVT.SimpleTy != MVT::i32)
8443 return Register();
8444 if ((!Subtarget->isThumb())) {
8445 return fastEmitInst_ri(MachineInstOpcode: ARM::ADDri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8446 }
8447 return Register();
8448}
8449
8450Register fastEmit_ISD_ADD_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8451 switch (VT.SimpleTy) {
8452 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8453 default: return Register();
8454 }
8455}
8456
8457// FastEmit functions for ISD::AND.
8458
8459Register fastEmit_ISD_AND_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8460 if (RetVT.SimpleTy != MVT::i32)
8461 return Register();
8462 if ((!Subtarget->isThumb())) {
8463 return fastEmitInst_ri(MachineInstOpcode: ARM::ANDri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8464 }
8465 return Register();
8466}
8467
8468Register fastEmit_ISD_AND_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8469 switch (VT.SimpleTy) {
8470 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8471 default: return Register();
8472 }
8473}
8474
8475// FastEmit functions for ISD::OR.
8476
8477Register fastEmit_ISD_OR_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8478 if (RetVT.SimpleTy != MVT::i32)
8479 return Register();
8480 if ((!Subtarget->isThumb())) {
8481 return fastEmitInst_ri(MachineInstOpcode: ARM::ORRri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8482 }
8483 return Register();
8484}
8485
8486Register fastEmit_ISD_OR_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8487 switch (VT.SimpleTy) {
8488 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8489 default: return Register();
8490 }
8491}
8492
8493// FastEmit functions for ISD::SUB.
8494
8495Register fastEmit_ISD_SUB_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8496 if (RetVT.SimpleTy != MVT::i32)
8497 return Register();
8498 if ((!Subtarget->isThumb())) {
8499 return fastEmitInst_ri(MachineInstOpcode: ARM::SUBri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8500 }
8501 return Register();
8502}
8503
8504Register fastEmit_ISD_SUB_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8505 switch (VT.SimpleTy) {
8506 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8507 default: return Register();
8508 }
8509}
8510
8511// FastEmit functions for ISD::XOR.
8512
8513Register fastEmit_ISD_XOR_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8514 if (RetVT.SimpleTy != MVT::i32)
8515 return Register();
8516 if ((!Subtarget->isThumb())) {
8517 return fastEmitInst_ri(MachineInstOpcode: ARM::EORri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8518 }
8519 return Register();
8520}
8521
8522Register fastEmit_ISD_XOR_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8523 switch (VT.SimpleTy) {
8524 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8525 default: return Register();
8526 }
8527}
8528
8529// Top-level FastEmit function.
8530
8531Register fastEmit_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8532 switch (Opcode) {
8533 case ARMISD::CMN: return fastEmit_ARMISD_CMN_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8534 case ARMISD::CMP: return fastEmit_ARMISD_CMP_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8535 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8536 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8537 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8538 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8539 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8540 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8541 default: return Register();
8542 }
8543}
8544
8545// FastEmit functions for ARMISD::VSHLIMM.
8546
8547Register fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
8548 if (RetVT.SimpleTy != MVT::v16i8)
8549 return Register();
8550 if ((Subtarget->hasMVEIntegerOps())) {
8551 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHL_immi8, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8552 }
8553 return Register();
8554}
8555
8556Register fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8557 switch (VT.SimpleTy) {
8558 case MVT::v16i8: return fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, imm1);
8559 default: return Register();
8560 }
8561}
8562
8563// FastEmit functions for ARMISD::VSHRsIMM.
8564
8565Register fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
8566 if (RetVT.SimpleTy != MVT::v16i8)
8567 return Register();
8568 if ((Subtarget->hasMVEIntegerOps())) {
8569 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_imms8, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8570 }
8571 return Register();
8572}
8573
8574Register fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8575 switch (VT.SimpleTy) {
8576 case MVT::v16i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, imm1);
8577 default: return Register();
8578 }
8579}
8580
8581// FastEmit functions for ARMISD::VSHRuIMM.
8582
8583Register fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
8584 if (RetVT.SimpleTy != MVT::v16i8)
8585 return Register();
8586 if ((Subtarget->hasMVEIntegerOps())) {
8587 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_immu8, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8588 }
8589 return Register();
8590}
8591
8592Register fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8593 switch (VT.SimpleTy) {
8594 case MVT::v16i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, imm1);
8595 default: return Register();
8596 }
8597}
8598
8599// FastEmit functions for ISD::ADD.
8600
8601Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
8602 if (RetVT.SimpleTy != MVT::i32)
8603 return Register();
8604 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
8605 return fastEmitInst_ri(MachineInstOpcode: ARM::tADDi3, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
8606 }
8607 return Register();
8608}
8609
8610Register fastEmit_ISD_ADD_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8611 switch (VT.SimpleTy) {
8612 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_7(RetVT, Op0, imm1);
8613 default: return Register();
8614 }
8615}
8616
8617// Top-level FastEmit function.
8618
8619Register fastEmit_ri_Predicate_imm0_7(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8620 switch (Opcode) {
8621 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
8622 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
8623 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
8624 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
8625 default: return Register();
8626 }
8627}
8628
8629// FastEmit functions for ISD::ADD.
8630
8631Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_255_expr(MVT RetVT, Register Op0, uint64_t imm1) {
8632 if (RetVT.SimpleTy != MVT::i32)
8633 return Register();
8634 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
8635 return fastEmitInst_ri(MachineInstOpcode: ARM::tADDi8, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
8636 }
8637 return Register();
8638}
8639
8640Register fastEmit_ISD_ADD_ri_Predicate_imm0_255_expr(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8641 switch (VT.SimpleTy) {
8642 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_255_expr(RetVT, Op0, imm1);
8643 default: return Register();
8644 }
8645}
8646
8647// Top-level FastEmit function.
8648
8649Register fastEmit_ri_Predicate_imm0_255_expr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8650 switch (Opcode) {
8651 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm0_255_expr(VT, RetVT, Op0, imm1);
8652 default: return Register();
8653 }
8654}
8655
8656// FastEmit functions for ARMISD::CMP.
8657
8658Register fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_imm0_255(MVT RetVT, Register Op0, uint64_t imm1) {
8659 if (RetVT.SimpleTy != MVT::i32)
8660 return Register();
8661 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
8662 return fastEmitInst_ri(MachineInstOpcode: ARM::tCMPi8, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
8663 }
8664 return Register();
8665}
8666
8667Register fastEmit_ARMISD_CMP_ri_Predicate_imm0_255(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8668 switch (VT.SimpleTy) {
8669 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_imm0_255(RetVT, Op0, imm1);
8670 default: return Register();
8671 }
8672}
8673
8674// FastEmit functions for ARMISD::CMPZ.
8675
8676Register fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_imm0_255(MVT RetVT, Register Op0, uint64_t imm1) {
8677 if (RetVT.SimpleTy != MVT::i32)
8678 return Register();
8679 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
8680 return fastEmitInst_ri(MachineInstOpcode: ARM::tCMPi8, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
8681 }
8682 return Register();
8683}
8684
8685Register fastEmit_ARMISD_CMPZ_ri_Predicate_imm0_255(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8686 switch (VT.SimpleTy) {
8687 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_imm0_255(RetVT, Op0, imm1);
8688 default: return Register();
8689 }
8690}
8691
8692// Top-level FastEmit function.
8693
8694Register fastEmit_ri_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8695 switch (Opcode) {
8696 case ARMISD::CMP: return fastEmit_ARMISD_CMP_ri_Predicate_imm0_255(VT, RetVT, Op0, imm1);
8697 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_ri_Predicate_imm0_255(VT, RetVT, Op0, imm1);
8698 default: return Register();
8699 }
8700}
8701
8702// FastEmit functions for ARMISD::CMP.
8703
8704Register fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8705 if (RetVT.SimpleTy != MVT::i32)
8706 return Register();
8707 if ((Subtarget->isThumb2())) {
8708 return fastEmitInst_ri(MachineInstOpcode: ARM::t2CMPri, RC: &ARM::GPRnopcRegClass, Op0, Imm: imm1);
8709 }
8710 return Register();
8711}
8712
8713Register fastEmit_ARMISD_CMP_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8714 switch (VT.SimpleTy) {
8715 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8716 default: return Register();
8717 }
8718}
8719
8720// FastEmit functions for ARMISD::CMPZ.
8721
8722Register fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8723 if (RetVT.SimpleTy != MVT::i32)
8724 return Register();
8725 if ((Subtarget->isThumb2())) {
8726 return fastEmitInst_ri(MachineInstOpcode: ARM::t2CMPri, RC: &ARM::GPRnopcRegClass, Op0, Imm: imm1);
8727 }
8728 return Register();
8729}
8730
8731Register fastEmit_ARMISD_CMPZ_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8732 switch (VT.SimpleTy) {
8733 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8734 default: return Register();
8735 }
8736}
8737
8738// FastEmit functions for ISD::ADD.
8739
8740Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8741 if (RetVT.SimpleTy != MVT::i32)
8742 return Register();
8743 if ((Subtarget->isThumb2())) {
8744 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ADDri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8745 }
8746 return Register();
8747}
8748
8749Register fastEmit_ISD_ADD_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8750 switch (VT.SimpleTy) {
8751 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8752 default: return Register();
8753 }
8754}
8755
8756// FastEmit functions for ISD::AND.
8757
8758Register fastEmit_ISD_AND_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8759 if (RetVT.SimpleTy != MVT::i32)
8760 return Register();
8761 if ((Subtarget->isThumb2())) {
8762 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ANDri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8763 }
8764 return Register();
8765}
8766
8767Register fastEmit_ISD_AND_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8768 switch (VT.SimpleTy) {
8769 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8770 default: return Register();
8771 }
8772}
8773
8774// FastEmit functions for ISD::OR.
8775
8776Register fastEmit_ISD_OR_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8777 if (RetVT.SimpleTy != MVT::i32)
8778 return Register();
8779 if ((Subtarget->isThumb2())) {
8780 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ORRri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8781 }
8782 return Register();
8783}
8784
8785Register fastEmit_ISD_OR_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8786 switch (VT.SimpleTy) {
8787 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8788 default: return Register();
8789 }
8790}
8791
8792// FastEmit functions for ISD::SUB.
8793
8794Register fastEmit_ISD_SUB_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8795 if (RetVT.SimpleTy != MVT::i32)
8796 return Register();
8797 if ((Subtarget->isThumb2())) {
8798 return fastEmitInst_ri(MachineInstOpcode: ARM::t2SUBri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8799 }
8800 return Register();
8801}
8802
8803Register fastEmit_ISD_SUB_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8804 switch (VT.SimpleTy) {
8805 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8806 default: return Register();
8807 }
8808}
8809
8810// FastEmit functions for ISD::XOR.
8811
8812Register fastEmit_ISD_XOR_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8813 if (RetVT.SimpleTy != MVT::i32)
8814 return Register();
8815 if ((Subtarget->isThumb2())) {
8816 return fastEmitInst_ri(MachineInstOpcode: ARM::t2EORri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8817 }
8818 return Register();
8819}
8820
8821Register fastEmit_ISD_XOR_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8822 switch (VT.SimpleTy) {
8823 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8824 default: return Register();
8825 }
8826}
8827
8828// Top-level FastEmit function.
8829
8830Register fastEmit_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8831 switch (Opcode) {
8832 case ARMISD::CMP: return fastEmit_ARMISD_CMP_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8833 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8834 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8835 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8836 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8837 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8838 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8839 default: return Register();
8840 }
8841}
8842
8843// FastEmit functions for ISD::ADD.
8844
8845Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_4095(MVT RetVT, Register Op0, uint64_t imm1) {
8846 if (RetVT.SimpleTy != MVT::i32)
8847 return Register();
8848 if ((Subtarget->isThumb2())) {
8849 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ADDri12, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8850 }
8851 return Register();
8852}
8853
8854Register fastEmit_ISD_ADD_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8855 switch (VT.SimpleTy) {
8856 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_4095(RetVT, Op0, imm1);
8857 default: return Register();
8858 }
8859}
8860
8861// FastEmit functions for ISD::SUB.
8862
8863Register fastEmit_ISD_SUB_MVT_i32_ri_Predicate_imm0_4095(MVT RetVT, Register Op0, uint64_t imm1) {
8864 if (RetVT.SimpleTy != MVT::i32)
8865 return Register();
8866 if ((Subtarget->isThumb2())) {
8867 return fastEmitInst_ri(MachineInstOpcode: ARM::t2SUBri12, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8868 }
8869 return Register();
8870}
8871
8872Register fastEmit_ISD_SUB_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8873 switch (VT.SimpleTy) {
8874 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_imm0_4095(RetVT, Op0, imm1);
8875 default: return Register();
8876 }
8877}
8878
8879// Top-level FastEmit function.
8880
8881Register fastEmit_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8882 switch (Opcode) {
8883 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm0_4095(VT, RetVT, Op0, imm1);
8884 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_imm0_4095(VT, RetVT, Op0, imm1);
8885 default: return Register();
8886 }
8887}
8888
8889// FastEmit functions for ISD::ROTR.
8890
8891Register fastEmit_ISD_ROTR_MVT_i32_ri_Predicate_imm1_31(MVT RetVT, Register Op0, uint64_t imm1) {
8892 if (RetVT.SimpleTy != MVT::i32)
8893 return Register();
8894 if ((Subtarget->isThumb2())) {
8895 return fastEmitInst_ri(MachineInstOpcode: ARM::t2RORri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8896 }
8897 return Register();
8898}
8899
8900Register fastEmit_ISD_ROTR_ri_Predicate_imm1_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8901 switch (VT.SimpleTy) {
8902 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_ri_Predicate_imm1_31(RetVT, Op0, imm1);
8903 default: return Register();
8904 }
8905}
8906
8907// FastEmit functions for ISD::SHL.
8908
8909Register fastEmit_ISD_SHL_MVT_i32_ri_Predicate_imm1_31(MVT RetVT, Register Op0, uint64_t imm1) {
8910 if (RetVT.SimpleTy != MVT::i32)
8911 return Register();
8912 if ((Subtarget->isThumb2())) {
8913 return fastEmitInst_ri(MachineInstOpcode: ARM::t2LSLri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8914 }
8915 return Register();
8916}
8917
8918Register fastEmit_ISD_SHL_ri_Predicate_imm1_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8919 switch (VT.SimpleTy) {
8920 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri_Predicate_imm1_31(RetVT, Op0, imm1);
8921 default: return Register();
8922 }
8923}
8924
8925// Top-level FastEmit function.
8926
8927Register fastEmit_ri_Predicate_imm1_31(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8928 switch (Opcode) {
8929 case ISD::ROTR: return fastEmit_ISD_ROTR_ri_Predicate_imm1_31(VT, RetVT, Op0, imm1);
8930 case ISD::SHL: return fastEmit_ISD_SHL_ri_Predicate_imm1_31(VT, RetVT, Op0, imm1);
8931 default: return Register();
8932 }
8933}
8934
8935// FastEmit functions for ARMISD::VQRSHRNsIMM.
8936
8937Register fastEmit_ARMISD_VQRSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8938 if (RetVT.SimpleTy != MVT::v8i8)
8939 return Register();
8940 if ((Subtarget->hasNEON())) {
8941 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8942 }
8943 return Register();
8944}
8945
8946Register fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8947 switch (VT.SimpleTy) {
8948 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8949 default: return Register();
8950 }
8951}
8952
8953// FastEmit functions for ARMISD::VQRSHRNsuIMM.
8954
8955Register fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8956 if (RetVT.SimpleTy != MVT::v8i8)
8957 return Register();
8958 if ((Subtarget->hasNEON())) {
8959 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRUNv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8960 }
8961 return Register();
8962}
8963
8964Register fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8965 switch (VT.SimpleTy) {
8966 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8967 default: return Register();
8968 }
8969}
8970
8971// FastEmit functions for ARMISD::VQRSHRNuIMM.
8972
8973Register fastEmit_ARMISD_VQRSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8974 if (RetVT.SimpleTy != MVT::v8i8)
8975 return Register();
8976 if ((Subtarget->hasNEON())) {
8977 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8978 }
8979 return Register();
8980}
8981
8982Register fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8983 switch (VT.SimpleTy) {
8984 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8985 default: return Register();
8986 }
8987}
8988
8989// FastEmit functions for ARMISD::VQSHRNsIMM.
8990
8991Register fastEmit_ARMISD_VQSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8992 if (RetVT.SimpleTy != MVT::v8i8)
8993 return Register();
8994 if ((Subtarget->hasNEON())) {
8995 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8996 }
8997 return Register();
8998}
8999
9000Register fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9001 switch (VT.SimpleTy) {
9002 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
9003 default: return Register();
9004 }
9005}
9006
9007// FastEmit functions for ARMISD::VQSHRNsuIMM.
9008
9009Register fastEmit_ARMISD_VQSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
9010 if (RetVT.SimpleTy != MVT::v8i8)
9011 return Register();
9012 if ((Subtarget->hasNEON())) {
9013 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRUNv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9014 }
9015 return Register();
9016}
9017
9018Register fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9019 switch (VT.SimpleTy) {
9020 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
9021 default: return Register();
9022 }
9023}
9024
9025// FastEmit functions for ARMISD::VQSHRNuIMM.
9026
9027Register fastEmit_ARMISD_VQSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
9028 if (RetVT.SimpleTy != MVT::v8i8)
9029 return Register();
9030 if ((Subtarget->hasNEON())) {
9031 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9032 }
9033 return Register();
9034}
9035
9036Register fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9037 switch (VT.SimpleTy) {
9038 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
9039 default: return Register();
9040 }
9041}
9042
9043// FastEmit functions for ARMISD::VRSHRNIMM.
9044
9045Register fastEmit_ARMISD_VRSHRNIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
9046 if (RetVT.SimpleTy != MVT::v8i8)
9047 return Register();
9048 if ((Subtarget->hasNEON())) {
9049 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRNv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9050 }
9051 return Register();
9052}
9053
9054Register fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9055 switch (VT.SimpleTy) {
9056 case MVT::v8i16: return fastEmit_ARMISD_VRSHRNIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
9057 default: return Register();
9058 }
9059}
9060
9061// Top-level FastEmit function.
9062
9063Register fastEmit_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9064 switch (Opcode) {
9065 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
9066 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
9067 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
9068 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
9069 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
9070 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
9071 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
9072 default: return Register();
9073 }
9074}
9075
9076// FastEmit functions for ARMISD::VQRSHRNsIMM.
9077
9078Register fastEmit_ARMISD_VQRSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
9079 if (RetVT.SimpleTy != MVT::v4i16)
9080 return Register();
9081 if ((Subtarget->hasNEON())) {
9082 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9083 }
9084 return Register();
9085}
9086
9087Register fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9088 switch (VT.SimpleTy) {
9089 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
9090 default: return Register();
9091 }
9092}
9093
9094// FastEmit functions for ARMISD::VQRSHRNsuIMM.
9095
9096Register fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
9097 if (RetVT.SimpleTy != MVT::v4i16)
9098 return Register();
9099 if ((Subtarget->hasNEON())) {
9100 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRUNv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9101 }
9102 return Register();
9103}
9104
9105Register fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9106 switch (VT.SimpleTy) {
9107 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
9108 default: return Register();
9109 }
9110}
9111
9112// FastEmit functions for ARMISD::VQRSHRNuIMM.
9113
9114Register fastEmit_ARMISD_VQRSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
9115 if (RetVT.SimpleTy != MVT::v4i16)
9116 return Register();
9117 if ((Subtarget->hasNEON())) {
9118 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9119 }
9120 return Register();
9121}
9122
9123Register fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9124 switch (VT.SimpleTy) {
9125 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
9126 default: return Register();
9127 }
9128}
9129
9130// FastEmit functions for ARMISD::VQSHRNsIMM.
9131
9132Register fastEmit_ARMISD_VQSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
9133 if (RetVT.SimpleTy != MVT::v4i16)
9134 return Register();
9135 if ((Subtarget->hasNEON())) {
9136 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9137 }
9138 return Register();
9139}
9140
9141Register fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9142 switch (VT.SimpleTy) {
9143 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
9144 default: return Register();
9145 }
9146}
9147
9148// FastEmit functions for ARMISD::VQSHRNsuIMM.
9149
9150Register fastEmit_ARMISD_VQSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
9151 if (RetVT.SimpleTy != MVT::v4i16)
9152 return Register();
9153 if ((Subtarget->hasNEON())) {
9154 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRUNv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9155 }
9156 return Register();
9157}
9158
9159Register fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9160 switch (VT.SimpleTy) {
9161 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
9162 default: return Register();
9163 }
9164}
9165
9166// FastEmit functions for ARMISD::VQSHRNuIMM.
9167
9168Register fastEmit_ARMISD_VQSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
9169 if (RetVT.SimpleTy != MVT::v4i16)
9170 return Register();
9171 if ((Subtarget->hasNEON())) {
9172 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9173 }
9174 return Register();
9175}
9176
9177Register fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9178 switch (VT.SimpleTy) {
9179 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
9180 default: return Register();
9181 }
9182}
9183
9184// FastEmit functions for ARMISD::VRSHRNIMM.
9185
9186Register fastEmit_ARMISD_VRSHRNIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
9187 if (RetVT.SimpleTy != MVT::v4i16)
9188 return Register();
9189 if ((Subtarget->hasNEON())) {
9190 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRNv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9191 }
9192 return Register();
9193}
9194
9195Register fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9196 switch (VT.SimpleTy) {
9197 case MVT::v4i32: return fastEmit_ARMISD_VRSHRNIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
9198 default: return Register();
9199 }
9200}
9201
9202// Top-level FastEmit function.
9203
9204Register fastEmit_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9205 switch (Opcode) {
9206 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9207 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9208 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9209 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9210 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9211 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9212 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9213 default: return Register();
9214 }
9215}
9216
9217// FastEmit functions for ARMISD::VQRSHRNsIMM.
9218
9219Register fastEmit_ARMISD_VQRSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9220 if (RetVT.SimpleTy != MVT::v2i32)
9221 return Register();
9222 if ((Subtarget->hasNEON())) {
9223 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9224 }
9225 return Register();
9226}
9227
9228Register fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9229 switch (VT.SimpleTy) {
9230 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9231 default: return Register();
9232 }
9233}
9234
9235// FastEmit functions for ARMISD::VQRSHRNsuIMM.
9236
9237Register fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9238 if (RetVT.SimpleTy != MVT::v2i32)
9239 return Register();
9240 if ((Subtarget->hasNEON())) {
9241 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRUNv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9242 }
9243 return Register();
9244}
9245
9246Register fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9247 switch (VT.SimpleTy) {
9248 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9249 default: return Register();
9250 }
9251}
9252
9253// FastEmit functions for ARMISD::VQRSHRNuIMM.
9254
9255Register fastEmit_ARMISD_VQRSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9256 if (RetVT.SimpleTy != MVT::v2i32)
9257 return Register();
9258 if ((Subtarget->hasNEON())) {
9259 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9260 }
9261 return Register();
9262}
9263
9264Register fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9265 switch (VT.SimpleTy) {
9266 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9267 default: return Register();
9268 }
9269}
9270
9271// FastEmit functions for ARMISD::VQSHRNsIMM.
9272
9273Register fastEmit_ARMISD_VQSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9274 if (RetVT.SimpleTy != MVT::v2i32)
9275 return Register();
9276 if ((Subtarget->hasNEON())) {
9277 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9278 }
9279 return Register();
9280}
9281
9282Register fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9283 switch (VT.SimpleTy) {
9284 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9285 default: return Register();
9286 }
9287}
9288
9289// FastEmit functions for ARMISD::VQSHRNsuIMM.
9290
9291Register fastEmit_ARMISD_VQSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9292 if (RetVT.SimpleTy != MVT::v2i32)
9293 return Register();
9294 if ((Subtarget->hasNEON())) {
9295 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRUNv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9296 }
9297 return Register();
9298}
9299
9300Register fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9301 switch (VT.SimpleTy) {
9302 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9303 default: return Register();
9304 }
9305}
9306
9307// FastEmit functions for ARMISD::VQSHRNuIMM.
9308
9309Register fastEmit_ARMISD_VQSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9310 if (RetVT.SimpleTy != MVT::v2i32)
9311 return Register();
9312 if ((Subtarget->hasNEON())) {
9313 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9314 }
9315 return Register();
9316}
9317
9318Register fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9319 switch (VT.SimpleTy) {
9320 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9321 default: return Register();
9322 }
9323}
9324
9325// FastEmit functions for ARMISD::VRSHRNIMM.
9326
9327Register fastEmit_ARMISD_VRSHRNIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9328 if (RetVT.SimpleTy != MVT::v2i32)
9329 return Register();
9330 if ((Subtarget->hasNEON())) {
9331 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRNv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9332 }
9333 return Register();
9334}
9335
9336Register fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9337 switch (VT.SimpleTy) {
9338 case MVT::v2i64: return fastEmit_ARMISD_VRSHRNIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9339 default: return Register();
9340 }
9341}
9342
9343// Top-level FastEmit function.
9344
9345Register fastEmit_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9346 switch (Opcode) {
9347 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9348 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9349 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9350 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9351 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9352 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9353 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9354 default: return Register();
9355 }
9356}
9357
9358// FastEmit functions for ARMISD::VDUPLANE.
9359
9360Register fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri_Predicate_VectorIndex32(MVT RetVT, Register Op0, uint64_t imm1) {
9361 if (RetVT.SimpleTy != MVT::v16i8)
9362 return Register();
9363 if ((Subtarget->hasNEON())) {
9364 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN8q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
9365 }
9366 return Register();
9367}
9368
9369Register fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri_Predicate_VectorIndex32(MVT RetVT, Register Op0, uint64_t imm1) {
9370 if (RetVT.SimpleTy != MVT::v8i16)
9371 return Register();
9372 if ((Subtarget->hasNEON())) {
9373 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
9374 }
9375 return Register();
9376}
9377
9378Register fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri_Predicate_VectorIndex32(MVT RetVT, Register Op0, uint64_t imm1) {
9379 if (RetVT.SimpleTy != MVT::v4i32)
9380 return Register();
9381 if ((Subtarget->hasNEON())) {
9382 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
9383 }
9384 return Register();
9385}
9386
9387Register fastEmit_ARMISD_VDUPLANE_ri_Predicate_VectorIndex32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9388 switch (VT.SimpleTy) {
9389 case MVT::v8i8: return fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri_Predicate_VectorIndex32(RetVT, Op0, imm1);
9390 case MVT::v4i16: return fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri_Predicate_VectorIndex32(RetVT, Op0, imm1);
9391 case MVT::v2i32: return fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri_Predicate_VectorIndex32(RetVT, Op0, imm1);
9392 default: return Register();
9393 }
9394}
9395
9396// Top-level FastEmit function.
9397
9398Register fastEmit_ri_Predicate_VectorIndex32(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9399 switch (Opcode) {
9400 case ARMISD::VDUPLANE: return fastEmit_ARMISD_VDUPLANE_ri_Predicate_VectorIndex32(VT, RetVT, Op0, imm1);
9401 default: return Register();
9402 }
9403}
9404
9405// FastEmit functions for ARMISD::VSHLIMM.
9406
9407Register fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, Register Op0, uint64_t imm1) {
9408 if (RetVT.SimpleTy != MVT::v4i32)
9409 return Register();
9410 if ((Subtarget->hasMVEIntegerOps())) {
9411 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHL_immi32, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9412 }
9413 return Register();
9414}
9415
9416Register fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9417 switch (VT.SimpleTy) {
9418 case MVT::v4i32: return fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, imm1);
9419 default: return Register();
9420 }
9421}
9422
9423// FastEmit functions for ARMISD::VSHRsIMM.
9424
9425Register fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, Register Op0, uint64_t imm1) {
9426 if (RetVT.SimpleTy != MVT::v4i32)
9427 return Register();
9428 if ((Subtarget->hasMVEIntegerOps())) {
9429 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_imms32, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9430 }
9431 return Register();
9432}
9433
9434Register fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9435 switch (VT.SimpleTy) {
9436 case MVT::v4i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, imm1);
9437 default: return Register();
9438 }
9439}
9440
9441// FastEmit functions for ARMISD::VSHRuIMM.
9442
9443Register fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, Register Op0, uint64_t imm1) {
9444 if (RetVT.SimpleTy != MVT::v4i32)
9445 return Register();
9446 if ((Subtarget->hasMVEIntegerOps())) {
9447 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_immu32, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9448 }
9449 return Register();
9450}
9451
9452Register fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9453 switch (VT.SimpleTy) {
9454 case MVT::v4i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, imm1);
9455 default: return Register();
9456 }
9457}
9458
9459// Top-level FastEmit function.
9460
9461Register fastEmit_ri_Predicate_imm0_31(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9462 switch (Opcode) {
9463 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, imm1);
9464 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, imm1);
9465 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, imm1);
9466 default: return Register();
9467 }
9468}
9469
9470// FastEmit functions for ARMISD::VSHLIMM.
9471
9472Register fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, Register Op0, uint64_t imm1) {
9473 if (RetVT.SimpleTy != MVT::v8i16)
9474 return Register();
9475 if ((Subtarget->hasMVEIntegerOps())) {
9476 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHL_immi16, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9477 }
9478 return Register();
9479}
9480
9481Register fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9482 switch (VT.SimpleTy) {
9483 case MVT::v8i16: return fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, imm1);
9484 default: return Register();
9485 }
9486}
9487
9488// FastEmit functions for ARMISD::VSHRsIMM.
9489
9490Register fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, Register Op0, uint64_t imm1) {
9491 if (RetVT.SimpleTy != MVT::v8i16)
9492 return Register();
9493 if ((Subtarget->hasMVEIntegerOps())) {
9494 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_imms16, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9495 }
9496 return Register();
9497}
9498
9499Register fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9500 switch (VT.SimpleTy) {
9501 case MVT::v8i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, imm1);
9502 default: return Register();
9503 }
9504}
9505
9506// FastEmit functions for ARMISD::VSHRuIMM.
9507
9508Register fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, Register Op0, uint64_t imm1) {
9509 if (RetVT.SimpleTy != MVT::v8i16)
9510 return Register();
9511 if ((Subtarget->hasMVEIntegerOps())) {
9512 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_immu16, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9513 }
9514 return Register();
9515}
9516
9517Register fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9518 switch (VT.SimpleTy) {
9519 case MVT::v8i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, imm1);
9520 default: return Register();
9521 }
9522}
9523
9524// Top-level FastEmit function.
9525
9526Register fastEmit_ri_Predicate_imm0_15(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9527 switch (Opcode) {
9528 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, imm1);
9529 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, imm1);
9530 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, imm1);
9531 default: return Register();
9532 }
9533}
9534
9535// FastEmit functions for ISD::Constant.
9536
9537Register fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
9538 if (RetVT.SimpleTy != MVT::i32)
9539 return Register();
9540 if ((Subtarget->isThumb()) && (Subtarget->useMovt())) {
9541 return fastEmitInst_i(MachineInstOpcode: ARM::t2MOVi32imm, RC: &ARM::rGPRRegClass, Imm: imm0);
9542 }
9543 if ((!Subtarget->useMovt()) && (Subtarget->genExecuteOnly()) && (Subtarget->isThumb1Only())) {
9544 return fastEmitInst_i(MachineInstOpcode: ARM::tMOVi32imm, RC: &ARM::rGPRRegClass, Imm: imm0);
9545 }
9546 return Register();
9547}
9548
9549Register fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
9550 switch (VT.SimpleTy) {
9551 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
9552 default: return Register();
9553 }
9554}
9555
9556// Top-level FastEmit function.
9557
9558Register fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
9559 switch (Opcode) {
9560 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
9561 default: return Register();
9562 }
9563}
9564
9565