1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the ARM target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_mod_imm(int64_t Imm) {
12
13 return ARM_AM::getSOImmVal(Arg: Imm) != -1;
14
15}
16static bool Predicate_imm0_7(int64_t Imm) {
17
18 return Imm >= 0 && Imm < 8;
19
20}
21static bool Predicate_imm0_255_expr(int64_t Imm) {
22 return Imm >= 0 && Imm < 256;
23}
24static bool Predicate_imm_sr(int64_t Imm) {
25
26 return Imm > 0 && Imm <= 32;
27
28}
29static bool Predicate_imm0_255(int64_t Imm) {
30 return Imm >= 0 && Imm < 256;
31}
32static bool Predicate_t2_so_imm(int64_t Imm) {
33
34 return ARM_AM::getT2SOImmVal(Arg: Imm) != -1;
35
36}
37static bool Predicate_imm0_4095(int64_t Imm) {
38
39 return Imm >= 0 && Imm < 4096;
40
41}
42static bool Predicate_imm1_31(int64_t Imm) {
43 return Imm > 0 && Imm < 32;
44}
45static bool Predicate_shr_imm8(int64_t Imm) {
46 return Imm > 0 && Imm <= 8;
47}
48static bool Predicate_shr_imm16(int64_t Imm) {
49 return Imm > 0 && Imm <= 16;
50}
51static bool Predicate_shr_imm32(int64_t Imm) {
52 return Imm > 0 && Imm <= 32;
53}
54static bool Predicate_VectorIndex32(int64_t Imm) {
55
56 return ((uint64_t)Imm) < 2;
57
58}
59static bool Predicate_imm0_31(int64_t Imm) {
60
61 return Imm >= 0 && Imm < 32;
62
63}
64static bool Predicate_mod_imm_not(int64_t Imm) {
65
66 return ARM_AM::getSOImmVal(Arg: ~(uint32_t)Imm) != -1;
67
68}
69static bool Predicate_imm0_65535(int64_t Imm) {
70
71 return Imm >= 0 && Imm < 65536;
72
73}
74static bool Predicate_t2_so_imm_neg(int64_t Imm) {
75
76 return Imm && ARM_AM::getT2SOImmVal(Arg: -(uint32_t)Imm) != -1;
77
78}
79static bool Predicate_imm0_15(int64_t Imm) {
80
81 return Imm >= 0 && Imm < 16;
82
83}
84
85
86// FastEmit functions for ISD::GET_FPENV.
87
88Register fastEmit_ISD_GET_FPENV_MVT_i32_(MVT RetVT) {
89 if (RetVT.SimpleTy != MVT::i32)
90 return Register();
91 return fastEmitInst_(MachineInstOpcode: ARM::VMRS, RC: &ARM::GPRnopcRegClass);
92}
93
94Register fastEmit_ISD_GET_FPENV_(MVT VT, MVT RetVT) {
95 switch (VT.SimpleTy) {
96 case MVT::i32: return fastEmit_ISD_GET_FPENV_MVT_i32_(RetVT);
97 default: return Register();
98 }
99}
100
101// FastEmit functions for ISD::GET_FPMODE.
102
103Register fastEmit_ISD_GET_FPMODE_MVT_i32_(MVT RetVT) {
104 if (RetVT.SimpleTy != MVT::i32)
105 return Register();
106 return fastEmitInst_(MachineInstOpcode: ARM::VMRS, RC: &ARM::GPRnopcRegClass);
107}
108
109Register fastEmit_ISD_GET_FPMODE_(MVT VT, MVT RetVT) {
110 switch (VT.SimpleTy) {
111 case MVT::i32: return fastEmit_ISD_GET_FPMODE_MVT_i32_(RetVT);
112 default: return Register();
113 }
114}
115
116// Top-level FastEmit function.
117
118Register fastEmit_(MVT VT, MVT RetVT, unsigned Opcode) override {
119 switch (Opcode) {
120 case ISD::GET_FPENV: return fastEmit_ISD_GET_FPENV_(VT, RetVT);
121 case ISD::GET_FPMODE: return fastEmit_ISD_GET_FPMODE_(VT, RetVT);
122 default: return Register();
123 }
124}
125
126// FastEmit functions for ARMISD::CALL.
127
128Register fastEmit_ARMISD_CALL_MVT_i32_r(MVT RetVT, Register Op0) {
129 if (RetVT.SimpleTy != MVT::isVoid)
130 return Register();
131 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
132 return fastEmitInst_r(MachineInstOpcode: ARM::BLX_noip, RC: &ARM::GPRnoipRegClass, Op0);
133 }
134 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( !MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
135 return fastEmitInst_r(MachineInstOpcode: ARM::BLX, RC: &ARM::GPRRegClass, Op0);
136 }
137 return Register();
138}
139
140Register fastEmit_ARMISD_CALL_r(MVT VT, MVT RetVT, Register Op0) {
141 switch (VT.SimpleTy) {
142 case MVT::i32: return fastEmit_ARMISD_CALL_MVT_i32_r(RetVT, Op0);
143 default: return Register();
144 }
145}
146
147// FastEmit functions for ARMISD::CALL_NOLINK.
148
149Register fastEmit_ARMISD_CALL_NOLINK_MVT_i32_r(MVT RetVT, Register Op0) {
150 if (RetVT.SimpleTy != MVT::isVoid)
151 return Register();
152 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
153 return fastEmitInst_r(MachineInstOpcode: ARM::tBX_CALL, RC: &ARM::tGPRRegClass, Op0);
154 }
155 if ((!Subtarget->isThumb()) && (!Subtarget->hasV4TOps())) {
156 return fastEmitInst_r(MachineInstOpcode: ARM::BMOVPCRX_CALL, RC: &ARM::tGPRRegClass, Op0);
157 }
158 if ((Subtarget->hasV4TOps()) && (!Subtarget->isThumb())) {
159 return fastEmitInst_r(MachineInstOpcode: ARM::BX_CALL, RC: &ARM::tGPRRegClass, Op0);
160 }
161 return Register();
162}
163
164Register fastEmit_ARMISD_CALL_NOLINK_r(MVT VT, MVT RetVT, Register Op0) {
165 switch (VT.SimpleTy) {
166 case MVT::i32: return fastEmit_ARMISD_CALL_NOLINK_MVT_i32_r(RetVT, Op0);
167 default: return Register();
168 }
169}
170
171// FastEmit functions for ARMISD::CALL_PRED.
172
173Register fastEmit_ARMISD_CALL_PRED_MVT_i32_r(MVT RetVT, Register Op0) {
174 if (RetVT.SimpleTy != MVT::isVoid)
175 return Register();
176 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
177 return fastEmitInst_r(MachineInstOpcode: ARM::BLX_pred_noip, RC: &ARM::GPRnoipRegClass, Op0);
178 }
179 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( !MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
180 return fastEmitInst_r(MachineInstOpcode: ARM::BLX_pred, RC: &ARM::GPRRegClass, Op0);
181 }
182 return Register();
183}
184
185Register fastEmit_ARMISD_CALL_PRED_r(MVT VT, MVT RetVT, Register Op0) {
186 switch (VT.SimpleTy) {
187 case MVT::i32: return fastEmit_ARMISD_CALL_PRED_MVT_i32_r(RetVT, Op0);
188 default: return Register();
189 }
190}
191
192// FastEmit functions for ARMISD::CMPFPEw0.
193
194Register fastEmit_ARMISD_CMPFPEw0_MVT_f16_r(MVT RetVT, Register Op0) {
195 if (RetVT.SimpleTy != MVT::i32)
196 return Register();
197 if ((Subtarget->hasFullFP16())) {
198 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPEZH, RC: &ARM::HPRRegClass, Op0);
199 }
200 return Register();
201}
202
203Register fastEmit_ARMISD_CMPFPEw0_MVT_f32_r(MVT RetVT, Register Op0) {
204 if (RetVT.SimpleTy != MVT::i32)
205 return Register();
206 if ((Subtarget->hasVFP2Base())) {
207 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPEZS, RC: &ARM::SPRRegClass, Op0);
208 }
209 return Register();
210}
211
212Register fastEmit_ARMISD_CMPFPEw0_MVT_f64_r(MVT RetVT, Register Op0) {
213 if (RetVT.SimpleTy != MVT::i32)
214 return Register();
215 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
216 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPEZD, RC: &ARM::DPRRegClass, Op0);
217 }
218 return Register();
219}
220
221Register fastEmit_ARMISD_CMPFPEw0_r(MVT VT, MVT RetVT, Register Op0) {
222 switch (VT.SimpleTy) {
223 case MVT::f16: return fastEmit_ARMISD_CMPFPEw0_MVT_f16_r(RetVT, Op0);
224 case MVT::f32: return fastEmit_ARMISD_CMPFPEw0_MVT_f32_r(RetVT, Op0);
225 case MVT::f64: return fastEmit_ARMISD_CMPFPEw0_MVT_f64_r(RetVT, Op0);
226 default: return Register();
227 }
228}
229
230// FastEmit functions for ARMISD::CMPFPw0.
231
232Register fastEmit_ARMISD_CMPFPw0_MVT_f16_r(MVT RetVT, Register Op0) {
233 if (RetVT.SimpleTy != MVT::i32)
234 return Register();
235 if ((Subtarget->hasFullFP16())) {
236 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPZH, RC: &ARM::HPRRegClass, Op0);
237 }
238 return Register();
239}
240
241Register fastEmit_ARMISD_CMPFPw0_MVT_f32_r(MVT RetVT, Register Op0) {
242 if (RetVT.SimpleTy != MVT::i32)
243 return Register();
244 if ((Subtarget->hasVFP2Base())) {
245 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPZS, RC: &ARM::SPRRegClass, Op0);
246 }
247 return Register();
248}
249
250Register fastEmit_ARMISD_CMPFPw0_MVT_f64_r(MVT RetVT, Register Op0) {
251 if (RetVT.SimpleTy != MVT::i32)
252 return Register();
253 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
254 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPZD, RC: &ARM::DPRRegClass, Op0);
255 }
256 return Register();
257}
258
259Register fastEmit_ARMISD_CMPFPw0_r(MVT VT, MVT RetVT, Register Op0) {
260 switch (VT.SimpleTy) {
261 case MVT::f16: return fastEmit_ARMISD_CMPFPw0_MVT_f16_r(RetVT, Op0);
262 case MVT::f32: return fastEmit_ARMISD_CMPFPw0_MVT_f32_r(RetVT, Op0);
263 case MVT::f64: return fastEmit_ARMISD_CMPFPw0_MVT_f64_r(RetVT, Op0);
264 default: return Register();
265 }
266}
267
268// FastEmit functions for ARMISD::VADDVs.
269
270Register fastEmit_ARMISD_VADDVs_MVT_v16i8_r(MVT RetVT, Register Op0) {
271 if (RetVT.SimpleTy != MVT::i32)
272 return Register();
273 if ((Subtarget->hasMVEIntegerOps())) {
274 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVs8no_acc, RC: &ARM::tGPREvenRegClass, Op0);
275 }
276 return Register();
277}
278
279Register fastEmit_ARMISD_VADDVs_MVT_v8i16_r(MVT RetVT, Register Op0) {
280 if (RetVT.SimpleTy != MVT::i32)
281 return Register();
282 if ((Subtarget->hasMVEIntegerOps())) {
283 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVs16no_acc, RC: &ARM::tGPREvenRegClass, Op0);
284 }
285 return Register();
286}
287
288Register fastEmit_ARMISD_VADDVs_MVT_v4i32_r(MVT RetVT, Register Op0) {
289 if (RetVT.SimpleTy != MVT::i32)
290 return Register();
291 if ((Subtarget->hasMVEIntegerOps())) {
292 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVs32no_acc, RC: &ARM::tGPREvenRegClass, Op0);
293 }
294 return Register();
295}
296
297Register fastEmit_ARMISD_VADDVs_r(MVT VT, MVT RetVT, Register Op0) {
298 switch (VT.SimpleTy) {
299 case MVT::v16i8: return fastEmit_ARMISD_VADDVs_MVT_v16i8_r(RetVT, Op0);
300 case MVT::v8i16: return fastEmit_ARMISD_VADDVs_MVT_v8i16_r(RetVT, Op0);
301 case MVT::v4i32: return fastEmit_ARMISD_VADDVs_MVT_v4i32_r(RetVT, Op0);
302 default: return Register();
303 }
304}
305
306// FastEmit functions for ARMISD::VADDVu.
307
308Register fastEmit_ARMISD_VADDVu_MVT_v16i8_r(MVT RetVT, Register Op0) {
309 if (RetVT.SimpleTy != MVT::i32)
310 return Register();
311 if ((Subtarget->hasMVEIntegerOps())) {
312 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu8no_acc, RC: &ARM::tGPREvenRegClass, Op0);
313 }
314 return Register();
315}
316
317Register fastEmit_ARMISD_VADDVu_MVT_v8i16_r(MVT RetVT, Register Op0) {
318 if (RetVT.SimpleTy != MVT::i32)
319 return Register();
320 if ((Subtarget->hasMVEIntegerOps())) {
321 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu16no_acc, RC: &ARM::tGPREvenRegClass, Op0);
322 }
323 return Register();
324}
325
326Register fastEmit_ARMISD_VADDVu_MVT_v4i32_r(MVT RetVT, Register Op0) {
327 if (RetVT.SimpleTy != MVT::i32)
328 return Register();
329 if ((Subtarget->hasMVEIntegerOps())) {
330 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu32no_acc, RC: &ARM::tGPREvenRegClass, Op0);
331 }
332 return Register();
333}
334
335Register fastEmit_ARMISD_VADDVu_r(MVT VT, MVT RetVT, Register Op0) {
336 switch (VT.SimpleTy) {
337 case MVT::v16i8: return fastEmit_ARMISD_VADDVu_MVT_v16i8_r(RetVT, Op0);
338 case MVT::v8i16: return fastEmit_ARMISD_VADDVu_MVT_v8i16_r(RetVT, Op0);
339 case MVT::v4i32: return fastEmit_ARMISD_VADDVu_MVT_v4i32_r(RetVT, Op0);
340 default: return Register();
341 }
342}
343
344// FastEmit functions for ARMISD::VDUP.
345
346Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i8_r(Register Op0) {
347 if ((Subtarget->hasNEON())) {
348 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP8d, RC: &ARM::DPRRegClass, Op0);
349 }
350 return Register();
351}
352
353Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v16i8_r(Register Op0) {
354 if ((Subtarget->hasMVEIntegerOps())) {
355 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP8, RC: &ARM::MQPRRegClass, Op0);
356 }
357 if ((Subtarget->hasNEON())) {
358 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP8q, RC: &ARM::QPRRegClass, Op0);
359 }
360 return Register();
361}
362
363Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i16_r(Register Op0) {
364 if ((Subtarget->hasNEON())) {
365 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP16d, RC: &ARM::DPRRegClass, Op0);
366 }
367 return Register();
368}
369
370Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i16_r(Register Op0) {
371 if ((Subtarget->hasMVEIntegerOps())) {
372 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP16, RC: &ARM::MQPRRegClass, Op0);
373 }
374 if ((Subtarget->hasNEON())) {
375 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP16q, RC: &ARM::QPRRegClass, Op0);
376 }
377 return Register();
378}
379
380Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v2i32_r(Register Op0) {
381 if ((!Subtarget->hasSlowVDUP32()) && (Subtarget->hasNEON())) {
382 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP32d, RC: &ARM::DPRRegClass, Op0);
383 }
384 return Register();
385}
386
387Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i32_r(Register Op0) {
388 if ((Subtarget->hasMVEIntegerOps())) {
389 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP32, RC: &ARM::MQPRRegClass, Op0);
390 }
391 if ((Subtarget->hasNEON())) {
392 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP32q, RC: &ARM::QPRRegClass, Op0);
393 }
394 return Register();
395}
396
397Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8f16_r(Register Op0) {
398 if ((Subtarget->hasMVEIntegerOps())) {
399 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP16, RC: &ARM::MQPRRegClass, Op0);
400 }
401 return Register();
402}
403
404Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4f32_r(Register Op0) {
405 if ((Subtarget->hasMVEIntegerOps())) {
406 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP32, RC: &ARM::MQPRRegClass, Op0);
407 }
408 return Register();
409}
410
411Register fastEmit_ARMISD_VDUP_MVT_i32_r(MVT RetVT, Register Op0) {
412switch (RetVT.SimpleTy) {
413 case MVT::v8i8: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i8_r(Op0);
414 case MVT::v16i8: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v16i8_r(Op0);
415 case MVT::v4i16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i16_r(Op0);
416 case MVT::v8i16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i16_r(Op0);
417 case MVT::v2i32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v2i32_r(Op0);
418 case MVT::v4i32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i32_r(Op0);
419 case MVT::v8f16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8f16_r(Op0);
420 case MVT::v4f32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4f32_r(Op0);
421 default: return Register();
422}
423}
424
425Register fastEmit_ARMISD_VDUP_r(MVT VT, MVT RetVT, Register Op0) {
426 switch (VT.SimpleTy) {
427 case MVT::i32: return fastEmit_ARMISD_VDUP_MVT_i32_r(RetVT, Op0);
428 default: return Register();
429 }
430}
431
432// FastEmit functions for ARMISD::VMOVSR.
433
434Register fastEmit_ARMISD_VMOVSR_MVT_i32_r(MVT RetVT, Register Op0) {
435 if (RetVT.SimpleTy != MVT::f32)
436 return Register();
437 if ((Subtarget->hasFPRegs()) && (Subtarget->preferVMOVSR() ||!Subtarget->useNEONForSinglePrecisionFP())) {
438 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVSR, RC: &ARM::SPRRegClass, Op0);
439 }
440 return Register();
441}
442
443Register fastEmit_ARMISD_VMOVSR_r(MVT VT, MVT RetVT, Register Op0) {
444 switch (VT.SimpleTy) {
445 case MVT::i32: return fastEmit_ARMISD_VMOVSR_MVT_i32_r(RetVT, Op0);
446 default: return Register();
447 }
448}
449
450// FastEmit functions for ARMISD::VMOVhr.
451
452Register fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_bf16_r(Register Op0) {
453 if ((Subtarget->hasFPRegs16())) {
454 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVHR, RC: &ARM::HPRRegClass, Op0);
455 }
456 return Register();
457}
458
459Register fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_f16_r(Register Op0) {
460 if ((Subtarget->hasFPRegs16())) {
461 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVHR, RC: &ARM::HPRRegClass, Op0);
462 }
463 return Register();
464}
465
466Register fastEmit_ARMISD_VMOVhr_MVT_i32_r(MVT RetVT, Register Op0) {
467switch (RetVT.SimpleTy) {
468 case MVT::bf16: return fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_bf16_r(Op0);
469 case MVT::f16: return fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_f16_r(Op0);
470 default: return Register();
471}
472}
473
474Register fastEmit_ARMISD_VMOVhr_r(MVT VT, MVT RetVT, Register Op0) {
475 switch (VT.SimpleTy) {
476 case MVT::i32: return fastEmit_ARMISD_VMOVhr_MVT_i32_r(RetVT, Op0);
477 default: return Register();
478 }
479}
480
481// FastEmit functions for ARMISD::VMOVrh.
482
483Register fastEmit_ARMISD_VMOVrh_MVT_bf16_r(MVT RetVT, Register Op0) {
484 if (RetVT.SimpleTy != MVT::i32)
485 return Register();
486 if ((Subtarget->hasFPRegs16())) {
487 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVRH, RC: &ARM::rGPRRegClass, Op0);
488 }
489 return Register();
490}
491
492Register fastEmit_ARMISD_VMOVrh_MVT_f16_r(MVT RetVT, Register Op0) {
493 if (RetVT.SimpleTy != MVT::i32)
494 return Register();
495 if ((Subtarget->hasFPRegs16())) {
496 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVRH, RC: &ARM::rGPRRegClass, Op0);
497 }
498 return Register();
499}
500
501Register fastEmit_ARMISD_VMOVrh_r(MVT VT, MVT RetVT, Register Op0) {
502 switch (VT.SimpleTy) {
503 case MVT::bf16: return fastEmit_ARMISD_VMOVrh_MVT_bf16_r(RetVT, Op0);
504 case MVT::f16: return fastEmit_ARMISD_VMOVrh_MVT_f16_r(RetVT, Op0);
505 default: return Register();
506 }
507}
508
509// FastEmit functions for ARMISD::VREV16.
510
511Register fastEmit_ARMISD_VREV16_MVT_v8i8_r(MVT RetVT, Register Op0) {
512 if (RetVT.SimpleTy != MVT::v8i8)
513 return Register();
514 if ((Subtarget->hasNEON())) {
515 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
516 }
517 return Register();
518}
519
520Register fastEmit_ARMISD_VREV16_MVT_v16i8_r(MVT RetVT, Register Op0) {
521 if (RetVT.SimpleTy != MVT::v16i8)
522 return Register();
523 if ((Subtarget->hasMVEIntegerOps())) {
524 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
525 }
526 if ((Subtarget->hasNEON())) {
527 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
528 }
529 return Register();
530}
531
532Register fastEmit_ARMISD_VREV16_r(MVT VT, MVT RetVT, Register Op0) {
533 switch (VT.SimpleTy) {
534 case MVT::v8i8: return fastEmit_ARMISD_VREV16_MVT_v8i8_r(RetVT, Op0);
535 case MVT::v16i8: return fastEmit_ARMISD_VREV16_MVT_v16i8_r(RetVT, Op0);
536 default: return Register();
537 }
538}
539
540// FastEmit functions for ARMISD::VREV32.
541
542Register fastEmit_ARMISD_VREV32_MVT_v8i8_r(MVT RetVT, Register Op0) {
543 if (RetVT.SimpleTy != MVT::v8i8)
544 return Register();
545 if ((Subtarget->hasNEON())) {
546 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
547 }
548 return Register();
549}
550
551Register fastEmit_ARMISD_VREV32_MVT_v16i8_r(MVT RetVT, Register Op0) {
552 if (RetVT.SimpleTy != MVT::v16i8)
553 return Register();
554 if ((Subtarget->hasMVEIntegerOps())) {
555 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
556 }
557 if ((Subtarget->hasNEON())) {
558 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
559 }
560 return Register();
561}
562
563Register fastEmit_ARMISD_VREV32_MVT_v4i16_r(MVT RetVT, Register Op0) {
564 if (RetVT.SimpleTy != MVT::v4i16)
565 return Register();
566 if ((Subtarget->hasNEON())) {
567 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
568 }
569 return Register();
570}
571
572Register fastEmit_ARMISD_VREV32_MVT_v8i16_r(MVT RetVT, Register Op0) {
573 if (RetVT.SimpleTy != MVT::v8i16)
574 return Register();
575 if ((Subtarget->hasMVEIntegerOps())) {
576 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
577 }
578 if ((Subtarget->hasNEON())) {
579 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
580 }
581 return Register();
582}
583
584Register fastEmit_ARMISD_VREV32_MVT_v4f16_r(MVT RetVT, Register Op0) {
585 if (RetVT.SimpleTy != MVT::v4f16)
586 return Register();
587 if ((Subtarget->hasNEON())) {
588 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
589 }
590 return Register();
591}
592
593Register fastEmit_ARMISD_VREV32_MVT_v8f16_r(MVT RetVT, Register Op0) {
594 if (RetVT.SimpleTy != MVT::v8f16)
595 return Register();
596 if ((Subtarget->hasMVEIntegerOps())) {
597 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
598 }
599 if ((Subtarget->hasNEON())) {
600 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
601 }
602 return Register();
603}
604
605Register fastEmit_ARMISD_VREV32_MVT_v4bf16_r(MVT RetVT, Register Op0) {
606 if (RetVT.SimpleTy != MVT::v4bf16)
607 return Register();
608 if ((Subtarget->hasNEON())) {
609 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
610 }
611 return Register();
612}
613
614Register fastEmit_ARMISD_VREV32_MVT_v8bf16_r(MVT RetVT, Register Op0) {
615 if (RetVT.SimpleTy != MVT::v8bf16)
616 return Register();
617 if ((Subtarget->hasNEON())) {
618 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
619 }
620 return Register();
621}
622
623Register fastEmit_ARMISD_VREV32_r(MVT VT, MVT RetVT, Register Op0) {
624 switch (VT.SimpleTy) {
625 case MVT::v8i8: return fastEmit_ARMISD_VREV32_MVT_v8i8_r(RetVT, Op0);
626 case MVT::v16i8: return fastEmit_ARMISD_VREV32_MVT_v16i8_r(RetVT, Op0);
627 case MVT::v4i16: return fastEmit_ARMISD_VREV32_MVT_v4i16_r(RetVT, Op0);
628 case MVT::v8i16: return fastEmit_ARMISD_VREV32_MVT_v8i16_r(RetVT, Op0);
629 case MVT::v4f16: return fastEmit_ARMISD_VREV32_MVT_v4f16_r(RetVT, Op0);
630 case MVT::v8f16: return fastEmit_ARMISD_VREV32_MVT_v8f16_r(RetVT, Op0);
631 case MVT::v4bf16: return fastEmit_ARMISD_VREV32_MVT_v4bf16_r(RetVT, Op0);
632 case MVT::v8bf16: return fastEmit_ARMISD_VREV32_MVT_v8bf16_r(RetVT, Op0);
633 default: return Register();
634 }
635}
636
637// FastEmit functions for ARMISD::VREV64.
638
639Register fastEmit_ARMISD_VREV64_MVT_v8i8_r(MVT RetVT, Register Op0) {
640 if (RetVT.SimpleTy != MVT::v8i8)
641 return Register();
642 if ((Subtarget->hasNEON())) {
643 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
644 }
645 return Register();
646}
647
648Register fastEmit_ARMISD_VREV64_MVT_v16i8_r(MVT RetVT, Register Op0) {
649 if (RetVT.SimpleTy != MVT::v16i8)
650 return Register();
651 if ((Subtarget->hasMVEIntegerOps())) {
652 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
653 }
654 if ((Subtarget->hasNEON())) {
655 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
656 }
657 return Register();
658}
659
660Register fastEmit_ARMISD_VREV64_MVT_v4i16_r(MVT RetVT, Register Op0) {
661 if (RetVT.SimpleTy != MVT::v4i16)
662 return Register();
663 if ((Subtarget->hasNEON())) {
664 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
665 }
666 return Register();
667}
668
669Register fastEmit_ARMISD_VREV64_MVT_v8i16_r(MVT RetVT, Register Op0) {
670 if (RetVT.SimpleTy != MVT::v8i16)
671 return Register();
672 if ((Subtarget->hasMVEIntegerOps())) {
673 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
674 }
675 if ((Subtarget->hasNEON())) {
676 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
677 }
678 return Register();
679}
680
681Register fastEmit_ARMISD_VREV64_MVT_v2i32_r(MVT RetVT, Register Op0) {
682 if (RetVT.SimpleTy != MVT::v2i32)
683 return Register();
684 if ((Subtarget->hasNEON())) {
685 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
686 }
687 return Register();
688}
689
690Register fastEmit_ARMISD_VREV64_MVT_v4i32_r(MVT RetVT, Register Op0) {
691 if (RetVT.SimpleTy != MVT::v4i32)
692 return Register();
693 if ((Subtarget->hasMVEIntegerOps())) {
694 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
695 }
696 if ((Subtarget->hasNEON())) {
697 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
698 }
699 return Register();
700}
701
702Register fastEmit_ARMISD_VREV64_MVT_v4f16_r(MVT RetVT, Register Op0) {
703 if (RetVT.SimpleTy != MVT::v4f16)
704 return Register();
705 if ((Subtarget->hasNEON())) {
706 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
707 }
708 return Register();
709}
710
711Register fastEmit_ARMISD_VREV64_MVT_v8f16_r(MVT RetVT, Register Op0) {
712 if (RetVT.SimpleTy != MVT::v8f16)
713 return Register();
714 if ((Subtarget->hasMVEIntegerOps())) {
715 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
716 }
717 if ((Subtarget->hasNEON())) {
718 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
719 }
720 return Register();
721}
722
723Register fastEmit_ARMISD_VREV64_MVT_v4bf16_r(MVT RetVT, Register Op0) {
724 if (RetVT.SimpleTy != MVT::v4bf16)
725 return Register();
726 if ((Subtarget->hasNEON())) {
727 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
728 }
729 return Register();
730}
731
732Register fastEmit_ARMISD_VREV64_MVT_v8bf16_r(MVT RetVT, Register Op0) {
733 if (RetVT.SimpleTy != MVT::v8bf16)
734 return Register();
735 if ((Subtarget->hasNEON())) {
736 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
737 }
738 return Register();
739}
740
741Register fastEmit_ARMISD_VREV64_MVT_v2f32_r(MVT RetVT, Register Op0) {
742 if (RetVT.SimpleTy != MVT::v2f32)
743 return Register();
744 if ((Subtarget->hasNEON())) {
745 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
746 }
747 return Register();
748}
749
750Register fastEmit_ARMISD_VREV64_MVT_v4f32_r(MVT RetVT, Register Op0) {
751 if (RetVT.SimpleTy != MVT::v4f32)
752 return Register();
753 if ((Subtarget->hasMVEIntegerOps())) {
754 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
755 }
756 if ((Subtarget->hasNEON())) {
757 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
758 }
759 return Register();
760}
761
762Register fastEmit_ARMISD_VREV64_r(MVT VT, MVT RetVT, Register Op0) {
763 switch (VT.SimpleTy) {
764 case MVT::v8i8: return fastEmit_ARMISD_VREV64_MVT_v8i8_r(RetVT, Op0);
765 case MVT::v16i8: return fastEmit_ARMISD_VREV64_MVT_v16i8_r(RetVT, Op0);
766 case MVT::v4i16: return fastEmit_ARMISD_VREV64_MVT_v4i16_r(RetVT, Op0);
767 case MVT::v8i16: return fastEmit_ARMISD_VREV64_MVT_v8i16_r(RetVT, Op0);
768 case MVT::v2i32: return fastEmit_ARMISD_VREV64_MVT_v2i32_r(RetVT, Op0);
769 case MVT::v4i32: return fastEmit_ARMISD_VREV64_MVT_v4i32_r(RetVT, Op0);
770 case MVT::v4f16: return fastEmit_ARMISD_VREV64_MVT_v4f16_r(RetVT, Op0);
771 case MVT::v8f16: return fastEmit_ARMISD_VREV64_MVT_v8f16_r(RetVT, Op0);
772 case MVT::v4bf16: return fastEmit_ARMISD_VREV64_MVT_v4bf16_r(RetVT, Op0);
773 case MVT::v8bf16: return fastEmit_ARMISD_VREV64_MVT_v8bf16_r(RetVT, Op0);
774 case MVT::v2f32: return fastEmit_ARMISD_VREV64_MVT_v2f32_r(RetVT, Op0);
775 case MVT::v4f32: return fastEmit_ARMISD_VREV64_MVT_v4f32_r(RetVT, Op0);
776 default: return Register();
777 }
778}
779
780// FastEmit functions for ARMISD::WIN__DBZCHK.
781
782Register fastEmit_ARMISD_WIN__DBZCHK_MVT_i32_r(MVT RetVT, Register Op0) {
783 if (RetVT.SimpleTy != MVT::isVoid)
784 return Register();
785 return fastEmitInst_r(MachineInstOpcode: ARM::WIN__DBZCHK, RC: &ARM::tGPRRegClass, Op0);
786}
787
788Register fastEmit_ARMISD_WIN__DBZCHK_r(MVT VT, MVT RetVT, Register Op0) {
789 switch (VT.SimpleTy) {
790 case MVT::i32: return fastEmit_ARMISD_WIN__DBZCHK_MVT_i32_r(RetVT, Op0);
791 default: return Register();
792 }
793}
794
795// FastEmit functions for ARMISD::tSECALL.
796
797Register fastEmit_ARMISD_tSECALL_MVT_i32_r(MVT RetVT, Register Op0) {
798 if (RetVT.SimpleTy != MVT::isVoid)
799 return Register();
800 if ((Subtarget->has8MSecExt()) && (Subtarget->isThumb())) {
801 return fastEmitInst_r(MachineInstOpcode: ARM::tBLXNS_CALL, RC: &ARM::GPRnopcRegClass, Op0);
802 }
803 return Register();
804}
805
806Register fastEmit_ARMISD_tSECALL_r(MVT VT, MVT RetVT, Register Op0) {
807 switch (VT.SimpleTy) {
808 case MVT::i32: return fastEmit_ARMISD_tSECALL_MVT_i32_r(RetVT, Op0);
809 default: return Register();
810 }
811}
812
813// FastEmit functions for ISD::ABS.
814
815Register fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, Register Op0) {
816 if (RetVT.SimpleTy != MVT::v8i8)
817 return Register();
818 if ((Subtarget->hasNEON())) {
819 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv8i8, RC: &ARM::DPRRegClass, Op0);
820 }
821 return Register();
822}
823
824Register fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, Register Op0) {
825 if (RetVT.SimpleTy != MVT::v16i8)
826 return Register();
827 if ((Subtarget->hasMVEIntegerOps())) {
828 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSs8, RC: &ARM::MQPRRegClass, Op0);
829 }
830 if ((Subtarget->hasNEON())) {
831 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv16i8, RC: &ARM::QPRRegClass, Op0);
832 }
833 return Register();
834}
835
836Register fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, Register Op0) {
837 if (RetVT.SimpleTy != MVT::v4i16)
838 return Register();
839 if ((Subtarget->hasNEON())) {
840 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv4i16, RC: &ARM::DPRRegClass, Op0);
841 }
842 return Register();
843}
844
845Register fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, Register Op0) {
846 if (RetVT.SimpleTy != MVT::v8i16)
847 return Register();
848 if ((Subtarget->hasMVEIntegerOps())) {
849 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSs16, RC: &ARM::MQPRRegClass, Op0);
850 }
851 if ((Subtarget->hasNEON())) {
852 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv8i16, RC: &ARM::QPRRegClass, Op0);
853 }
854 return Register();
855}
856
857Register fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, Register Op0) {
858 if (RetVT.SimpleTy != MVT::v2i32)
859 return Register();
860 if ((Subtarget->hasNEON())) {
861 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv2i32, RC: &ARM::DPRRegClass, Op0);
862 }
863 return Register();
864}
865
866Register fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, Register Op0) {
867 if (RetVT.SimpleTy != MVT::v4i32)
868 return Register();
869 if ((Subtarget->hasMVEIntegerOps())) {
870 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSs32, RC: &ARM::MQPRRegClass, Op0);
871 }
872 if ((Subtarget->hasNEON())) {
873 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv4i32, RC: &ARM::QPRRegClass, Op0);
874 }
875 return Register();
876}
877
878Register fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, Register Op0) {
879 switch (VT.SimpleTy) {
880 case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0);
881 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
882 case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0);
883 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
884 case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0);
885 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
886 default: return Register();
887 }
888}
889
890// FastEmit functions for ISD::ANY_EXTEND.
891
892Register fastEmit_ISD_ANY_EXTEND_MVT_v8i8_r(MVT RetVT, Register Op0) {
893 if (RetVT.SimpleTy != MVT::v8i16)
894 return Register();
895 if ((Subtarget->hasNEON())) {
896 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv8i16, RC: &ARM::QPRRegClass, Op0);
897 }
898 return Register();
899}
900
901Register fastEmit_ISD_ANY_EXTEND_MVT_v4i16_r(MVT RetVT, Register Op0) {
902 if (RetVT.SimpleTy != MVT::v4i32)
903 return Register();
904 if ((Subtarget->hasNEON())) {
905 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv4i32, RC: &ARM::QPRRegClass, Op0);
906 }
907 return Register();
908}
909
910Register fastEmit_ISD_ANY_EXTEND_MVT_v2i32_r(MVT RetVT, Register Op0) {
911 if (RetVT.SimpleTy != MVT::v2i64)
912 return Register();
913 if ((Subtarget->hasNEON())) {
914 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv2i64, RC: &ARM::QPRRegClass, Op0);
915 }
916 return Register();
917}
918
919Register fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
920 switch (VT.SimpleTy) {
921 case MVT::v8i8: return fastEmit_ISD_ANY_EXTEND_MVT_v8i8_r(RetVT, Op0);
922 case MVT::v4i16: return fastEmit_ISD_ANY_EXTEND_MVT_v4i16_r(RetVT, Op0);
923 case MVT::v2i32: return fastEmit_ISD_ANY_EXTEND_MVT_v2i32_r(RetVT, Op0);
924 default: return Register();
925 }
926}
927
928// FastEmit functions for ISD::BITCAST.
929
930Register fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, Register Op0) {
931 if (RetVT.SimpleTy != MVT::f32)
932 return Register();
933 if ((Subtarget->hasFPRegs()) && (Subtarget->preferVMOVSR() ||!Subtarget->useNEONForSinglePrecisionFP())) {
934 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVSR, RC: &ARM::SPRRegClass, Op0);
935 }
936 return Register();
937}
938
939Register fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, Register Op0) {
940 if (RetVT.SimpleTy != MVT::i32)
941 return Register();
942 if ((Subtarget->hasFPRegs())) {
943 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVRS, RC: &ARM::GPRRegClass, Op0);
944 }
945 return Register();
946}
947
948Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Register Op0) {
949 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
950 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
951 }
952 return Register();
953}
954
955Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Register Op0) {
956 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
957 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
958 }
959 return Register();
960}
961
962Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Register Op0) {
963 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
964 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
965 }
966 return Register();
967}
968
969Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Register Op0) {
970 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
971 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
972 }
973 return Register();
974}
975
976Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Register Op0) {
977 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
978 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
979 }
980 return Register();
981}
982
983Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Register Op0) {
984 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
985 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
986 }
987 return Register();
988}
989
990Register fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, Register Op0) {
991switch (RetVT.SimpleTy) {
992 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0);
993 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0);
994 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0);
995 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0);
996 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Op0);
997 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0);
998 default: return Register();
999}
1000}
1001
1002Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Register Op0) {
1003 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1004 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
1005 }
1006 return Register();
1007}
1008
1009Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Register Op0) {
1010 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1011 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1012 }
1013 return Register();
1014}
1015
1016Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Register Op0) {
1017 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1018 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1019 }
1020 return Register();
1021}
1022
1023Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Register Op0) {
1024 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1025 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
1026 }
1027 return Register();
1028}
1029
1030Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Register Op0) {
1031 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1032 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1033 }
1034 return Register();
1035}
1036
1037Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Register Op0) {
1038 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1039 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1040 }
1041 return Register();
1042}
1043
1044Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Register Op0) {
1045 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1046 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1047 }
1048 return Register();
1049}
1050
1051Register fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, Register Op0) {
1052switch (RetVT.SimpleTy) {
1053 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0);
1054 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0);
1055 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0);
1056 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0);
1057 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0);
1058 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Op0);
1059 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0);
1060 default: return Register();
1061}
1062}
1063
1064Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Register Op0) {
1065 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1066 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1067 }
1068 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1069 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1070 }
1071 return Register();
1072}
1073
1074Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Register Op0) {
1075 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1076 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1077 }
1078 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1079 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1080 }
1081 return Register();
1082}
1083
1084Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Register Op0) {
1085 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1086 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1087 }
1088 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1089 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1090 }
1091 return Register();
1092}
1093
1094Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Register Op0) {
1095 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1096 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1097 }
1098 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1099 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1100 }
1101 return Register();
1102}
1103
1104Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Register Op0) {
1105 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1106 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1107 }
1108 return Register();
1109}
1110
1111Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Register Op0) {
1112 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1113 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1114 }
1115 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1116 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1117 }
1118 return Register();
1119}
1120
1121Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Register Op0) {
1122 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1123 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1124 }
1125 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1126 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1127 }
1128 return Register();
1129}
1130
1131Register fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, Register Op0) {
1132switch (RetVT.SimpleTy) {
1133 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0);
1134 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0);
1135 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0);
1136 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0);
1137 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Op0);
1138 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0);
1139 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0);
1140 default: return Register();
1141}
1142}
1143
1144Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Register Op0) {
1145 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1146 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1147 }
1148 return Register();
1149}
1150
1151Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Register Op0) {
1152 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1153 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1154 }
1155 return Register();
1156}
1157
1158Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Register Op0) {
1159 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1160 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1161 }
1162 return Register();
1163}
1164
1165Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Register Op0) {
1166 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1167 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1168 }
1169 return Register();
1170}
1171
1172Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Register Op0) {
1173 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1174 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1175 }
1176 return Register();
1177}
1178
1179Register fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, Register Op0) {
1180switch (RetVT.SimpleTy) {
1181 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0);
1182 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0);
1183 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0);
1184 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0);
1185 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0);
1186 default: return Register();
1187}
1188}
1189
1190Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Register Op0) {
1191 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1192 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1193 }
1194 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1195 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1196 }
1197 return Register();
1198}
1199
1200Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Register Op0) {
1201 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1202 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1203 }
1204 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1205 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1206 }
1207 return Register();
1208}
1209
1210Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Register Op0) {
1211 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1212 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1213 }
1214 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1215 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1216 }
1217 return Register();
1218}
1219
1220Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Register Op0) {
1221 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1222 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1223 }
1224 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1225 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1226 }
1227 return Register();
1228}
1229
1230Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Register Op0) {
1231 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1232 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1233 }
1234 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1235 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1236 }
1237 return Register();
1238}
1239
1240Register fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, Register Op0) {
1241switch (RetVT.SimpleTy) {
1242 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0);
1243 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0);
1244 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0);
1245 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0);
1246 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0);
1247 default: return Register();
1248}
1249}
1250
1251Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Register Op0) {
1252 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1253 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1254 }
1255 return Register();
1256}
1257
1258Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Register Op0) {
1259 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1260 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1261 }
1262 return Register();
1263}
1264
1265Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Register Op0) {
1266 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1267 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1268 }
1269 return Register();
1270}
1271
1272Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Register Op0) {
1273 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1274 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1275 }
1276 return Register();
1277}
1278
1279Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Register Op0) {
1280 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1281 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1282 }
1283 return Register();
1284}
1285
1286Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Register Op0) {
1287 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1288 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1289 }
1290 return Register();
1291}
1292
1293Register fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, Register Op0) {
1294switch (RetVT.SimpleTy) {
1295 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0);
1296 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0);
1297 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0);
1298 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0);
1299 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0);
1300 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Op0);
1301 default: return Register();
1302}
1303}
1304
1305Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Register Op0) {
1306 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1307 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1308 }
1309 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1310 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1311 }
1312 return Register();
1313}
1314
1315Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Register Op0) {
1316 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1317 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1318 }
1319 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1320 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1321 }
1322 return Register();
1323}
1324
1325Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Register Op0) {
1326 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1327 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1328 }
1329 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1330 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1331 }
1332 return Register();
1333}
1334
1335Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Register Op0) {
1336 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1337 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1338 }
1339 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1340 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1341 }
1342 return Register();
1343}
1344
1345Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Register Op0) {
1346 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1347 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1348 }
1349 return Register();
1350}
1351
1352Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Register Op0) {
1353 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1354 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1355 }
1356 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1357 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1358 }
1359 return Register();
1360}
1361
1362Register fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, Register Op0) {
1363switch (RetVT.SimpleTy) {
1364 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0);
1365 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0);
1366 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0);
1367 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0);
1368 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Op0);
1369 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0);
1370 default: return Register();
1371}
1372}
1373
1374Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Register Op0) {
1375 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1376 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
1377 }
1378 return Register();
1379}
1380
1381Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Register Op0) {
1382 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1383 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1384 }
1385 return Register();
1386}
1387
1388Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Register Op0) {
1389 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1390 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1391 }
1392 return Register();
1393}
1394
1395Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Register Op0) {
1396 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1397 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1398 }
1399 return Register();
1400}
1401
1402Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Register Op0) {
1403 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1404 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1405 }
1406 return Register();
1407}
1408
1409Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Register Op0) {
1410 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1411 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1412 }
1413 return Register();
1414}
1415
1416Register fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, Register Op0) {
1417switch (RetVT.SimpleTy) {
1418 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0);
1419 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0);
1420 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0);
1421 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0);
1422 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Op0);
1423 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0);
1424 default: return Register();
1425}
1426}
1427
1428Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Register Op0) {
1429 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1430 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1431 }
1432 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1433 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1434 }
1435 return Register();
1436}
1437
1438Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Register Op0) {
1439 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1440 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1441 }
1442 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1443 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1444 }
1445 return Register();
1446}
1447
1448Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Register Op0) {
1449 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1450 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1451 }
1452 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1453 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1454 }
1455 return Register();
1456}
1457
1458Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Register Op0) {
1459 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1460 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1461 }
1462 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1463 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1464 }
1465 return Register();
1466}
1467
1468Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Register Op0) {
1469 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1470 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1471 }
1472 return Register();
1473}
1474
1475Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Register Op0) {
1476 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1477 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1478 }
1479 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1480 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1481 }
1482 return Register();
1483}
1484
1485Register fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, Register Op0) {
1486switch (RetVT.SimpleTy) {
1487 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0);
1488 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0);
1489 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0);
1490 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0);
1491 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Op0);
1492 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0);
1493 default: return Register();
1494}
1495}
1496
1497Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Register Op0) {
1498 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1499 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1500 }
1501 return Register();
1502}
1503
1504Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Register Op0) {
1505 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1506 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1507 }
1508 return Register();
1509}
1510
1511Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Register Op0) {
1512 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1513 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1514 }
1515 return Register();
1516}
1517
1518Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Register Op0) {
1519 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1520 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1521 }
1522 return Register();
1523}
1524
1525Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Register Op0) {
1526 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1527 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1528 }
1529 return Register();
1530}
1531
1532Register fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, Register Op0) {
1533switch (RetVT.SimpleTy) {
1534 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0);
1535 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0);
1536 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0);
1537 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0);
1538 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0);
1539 default: return Register();
1540}
1541}
1542
1543Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Register Op0) {
1544 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1545 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1546 }
1547 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1548 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1549 }
1550 return Register();
1551}
1552
1553Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Register Op0) {
1554 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1555 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1556 }
1557 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1558 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1559 }
1560 return Register();
1561}
1562
1563Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Register Op0) {
1564 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1565 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1566 }
1567 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1568 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1569 }
1570 return Register();
1571}
1572
1573Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Register Op0) {
1574 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1575 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1576 }
1577 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1578 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1579 }
1580 return Register();
1581}
1582
1583Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Register Op0) {
1584 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1585 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1586 }
1587 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1588 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1589 }
1590 return Register();
1591}
1592
1593Register fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, Register Op0) {
1594switch (RetVT.SimpleTy) {
1595 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0);
1596 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0);
1597 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0);
1598 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0);
1599 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0);
1600 default: return Register();
1601}
1602}
1603
1604Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Register Op0) {
1605 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1606 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1607 }
1608 return Register();
1609}
1610
1611Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Register Op0) {
1612 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1613 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1614 }
1615 return Register();
1616}
1617
1618Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Register Op0) {
1619 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1620 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1621 }
1622 return Register();
1623}
1624
1625Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Register Op0) {
1626 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1627 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1628 }
1629 return Register();
1630}
1631
1632Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Register Op0) {
1633 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1634 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1635 }
1636 return Register();
1637}
1638
1639Register fastEmit_ISD_BITCAST_MVT_v4bf16_r(MVT RetVT, Register Op0) {
1640switch (RetVT.SimpleTy) {
1641 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Op0);
1642 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Op0);
1643 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Op0);
1644 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Op0);
1645 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Op0);
1646 default: return Register();
1647}
1648}
1649
1650Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Register Op0) {
1651 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1652 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1653 }
1654 return Register();
1655}
1656
1657Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Register Op0) {
1658 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1659 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1660 }
1661 return Register();
1662}
1663
1664Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Register Op0) {
1665 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1666 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1667 }
1668 return Register();
1669}
1670
1671Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Register Op0) {
1672 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1673 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1674 }
1675 return Register();
1676}
1677
1678Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Register Op0) {
1679 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1680 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1681 }
1682 return Register();
1683}
1684
1685Register fastEmit_ISD_BITCAST_MVT_v8bf16_r(MVT RetVT, Register Op0) {
1686switch (RetVT.SimpleTy) {
1687 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Op0);
1688 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Op0);
1689 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Op0);
1690 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Op0);
1691 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Op0);
1692 default: return Register();
1693}
1694}
1695
1696Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Register Op0) {
1697 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1698 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1699 }
1700 return Register();
1701}
1702
1703Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Register Op0) {
1704 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1705 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1706 }
1707 return Register();
1708}
1709
1710Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Register Op0) {
1711 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1712 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1713 }
1714 return Register();
1715}
1716
1717Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Register Op0) {
1718 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1719 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1720 }
1721 return Register();
1722}
1723
1724Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Register Op0) {
1725 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1726 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1727 }
1728 return Register();
1729}
1730
1731Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Register Op0) {
1732 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1733 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1734 }
1735 return Register();
1736}
1737
1738Register fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, Register Op0) {
1739switch (RetVT.SimpleTy) {
1740 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0);
1741 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0);
1742 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0);
1743 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0);
1744 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0);
1745 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Op0);
1746 default: return Register();
1747}
1748}
1749
1750Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Register Op0) {
1751 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1752 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1753 }
1754 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1755 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1756 }
1757 return Register();
1758}
1759
1760Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Register Op0) {
1761 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1762 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1763 }
1764 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1765 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1766 }
1767 return Register();
1768}
1769
1770Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Register Op0) {
1771 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1772 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1773 }
1774 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1775 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1776 }
1777 return Register();
1778}
1779
1780Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Register Op0) {
1781 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1782 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1783 }
1784 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1785 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1786 }
1787 return Register();
1788}
1789
1790Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Register Op0) {
1791 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1792 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1793 }
1794 return Register();
1795}
1796
1797Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Register Op0) {
1798 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1799 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1800 }
1801 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1802 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1803 }
1804 return Register();
1805}
1806
1807Register fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, Register Op0) {
1808switch (RetVT.SimpleTy) {
1809 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0);
1810 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0);
1811 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0);
1812 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0);
1813 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Op0);
1814 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0);
1815 default: return Register();
1816}
1817}
1818
1819Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Register Op0) {
1820 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1821 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1822 }
1823 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1824 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1825 }
1826 return Register();
1827}
1828
1829Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Register Op0) {
1830 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1831 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1832 }
1833 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1834 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1835 }
1836 return Register();
1837}
1838
1839Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Register Op0) {
1840 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1841 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1842 }
1843 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1844 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1845 }
1846 return Register();
1847}
1848
1849Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Register Op0) {
1850 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1851 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1852 }
1853 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1854 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1855 }
1856 return Register();
1857}
1858
1859Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Register Op0) {
1860 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1861 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1862 }
1863 return Register();
1864}
1865
1866Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Register Op0) {
1867 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1868 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1869 }
1870 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1871 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1872 }
1873 return Register();
1874}
1875
1876Register fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, Register Op0) {
1877switch (RetVT.SimpleTy) {
1878 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0);
1879 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0);
1880 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0);
1881 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0);
1882 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Op0);
1883 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0);
1884 default: return Register();
1885}
1886}
1887
1888Register fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, Register Op0) {
1889 switch (VT.SimpleTy) {
1890 case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0);
1891 case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0);
1892 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
1893 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0);
1894 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0);
1895 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0);
1896 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0);
1897 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0);
1898 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0);
1899 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0);
1900 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0);
1901 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0);
1902 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0);
1903 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v4bf16_r(RetVT, Op0);
1904 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v8bf16_r(RetVT, Op0);
1905 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0);
1906 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0);
1907 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0);
1908 default: return Register();
1909 }
1910}
1911
1912// FastEmit functions for ISD::BITREVERSE.
1913
1914Register fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, Register Op0) {
1915 if (RetVT.SimpleTy != MVT::i32)
1916 return Register();
1917 if ((Subtarget->isThumb2())) {
1918 return fastEmitInst_r(MachineInstOpcode: ARM::t2RBIT, RC: &ARM::rGPRRegClass, Op0);
1919 }
1920 if ((Subtarget->hasV6T2Ops()) && (!Subtarget->isThumb())) {
1921 return fastEmitInst_r(MachineInstOpcode: ARM::RBIT, RC: &ARM::GPRRegClass, Op0);
1922 }
1923 return Register();
1924}
1925
1926Register fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, Register Op0) {
1927 switch (VT.SimpleTy) {
1928 case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0);
1929 default: return Register();
1930 }
1931}
1932
1933// FastEmit functions for ISD::BRIND.
1934
1935Register fastEmit_ISD_BRIND_MVT_i32_r(MVT RetVT, Register Op0) {
1936 if (RetVT.SimpleTy != MVT::isVoid)
1937 return Register();
1938 if ((Subtarget->isThumb())) {
1939 return fastEmitInst_r(MachineInstOpcode: ARM::tBRIND, RC: &ARM::GPRRegClass, Op0);
1940 }
1941 if ((!Subtarget->isThumb()) && (!Subtarget->hasV4TOps())) {
1942 return fastEmitInst_r(MachineInstOpcode: ARM::MOVPCRX, RC: &ARM::GPRRegClass, Op0);
1943 }
1944 if ((Subtarget->hasV4TOps()) && (!Subtarget->isThumb())) {
1945 return fastEmitInst_r(MachineInstOpcode: ARM::BX, RC: &ARM::GPRRegClass, Op0);
1946 }
1947 return Register();
1948}
1949
1950Register fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, Register Op0) {
1951 switch (VT.SimpleTy) {
1952 case MVT::i32: return fastEmit_ISD_BRIND_MVT_i32_r(RetVT, Op0);
1953 default: return Register();
1954 }
1955}
1956
1957// FastEmit functions for ISD::BSWAP.
1958
1959Register fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, Register Op0) {
1960 if (RetVT.SimpleTy != MVT::i32)
1961 return Register();
1962 if ((Subtarget->isThumb2())) {
1963 return fastEmitInst_r(MachineInstOpcode: ARM::t2REV, RC: &ARM::rGPRRegClass, Op0);
1964 }
1965 if ((Subtarget->hasV6Ops()) && (Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
1966 return fastEmitInst_r(MachineInstOpcode: ARM::tREV, RC: &ARM::tGPRRegClass, Op0);
1967 }
1968 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
1969 return fastEmitInst_r(MachineInstOpcode: ARM::REV, RC: &ARM::GPRRegClass, Op0);
1970 }
1971 return Register();
1972}
1973
1974Register fastEmit_ISD_BSWAP_MVT_v8i16_r(MVT RetVT, Register Op0) {
1975 if (RetVT.SimpleTy != MVT::v8i16)
1976 return Register();
1977 if ((Subtarget->hasMVEIntegerOps())) {
1978 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1979 }
1980 return Register();
1981}
1982
1983Register fastEmit_ISD_BSWAP_MVT_v4i32_r(MVT RetVT, Register Op0) {
1984 if (RetVT.SimpleTy != MVT::v4i32)
1985 return Register();
1986 if ((Subtarget->hasMVEIntegerOps())) {
1987 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1988 }
1989 return Register();
1990}
1991
1992Register fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, Register Op0) {
1993 switch (VT.SimpleTy) {
1994 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
1995 case MVT::v8i16: return fastEmit_ISD_BSWAP_MVT_v8i16_r(RetVT, Op0);
1996 case MVT::v4i32: return fastEmit_ISD_BSWAP_MVT_v4i32_r(RetVT, Op0);
1997 default: return Register();
1998 }
1999}
2000
2001// FastEmit functions for ISD::CTLZ.
2002
2003Register fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, Register Op0) {
2004 if (RetVT.SimpleTy != MVT::i32)
2005 return Register();
2006 if ((Subtarget->isThumb2())) {
2007 return fastEmitInst_r(MachineInstOpcode: ARM::t2CLZ, RC: &ARM::rGPRRegClass, Op0);
2008 }
2009 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb())) {
2010 return fastEmitInst_r(MachineInstOpcode: ARM::CLZ, RC: &ARM::GPRRegClass, Op0);
2011 }
2012 return Register();
2013}
2014
2015Register fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, Register Op0) {
2016 if (RetVT.SimpleTy != MVT::v8i8)
2017 return Register();
2018 if ((Subtarget->hasNEON())) {
2019 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv8i8, RC: &ARM::DPRRegClass, Op0);
2020 }
2021 return Register();
2022}
2023
2024Register fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, Register Op0) {
2025 if (RetVT.SimpleTy != MVT::v16i8)
2026 return Register();
2027 if ((Subtarget->hasMVEIntegerOps())) {
2028 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLZs8, RC: &ARM::MQPRRegClass, Op0);
2029 }
2030 if ((Subtarget->hasNEON())) {
2031 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv16i8, RC: &ARM::QPRRegClass, Op0);
2032 }
2033 return Register();
2034}
2035
2036Register fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, Register Op0) {
2037 if (RetVT.SimpleTy != MVT::v4i16)
2038 return Register();
2039 if ((Subtarget->hasNEON())) {
2040 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv4i16, RC: &ARM::DPRRegClass, Op0);
2041 }
2042 return Register();
2043}
2044
2045Register fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, Register Op0) {
2046 if (RetVT.SimpleTy != MVT::v8i16)
2047 return Register();
2048 if ((Subtarget->hasMVEIntegerOps())) {
2049 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLZs16, RC: &ARM::MQPRRegClass, Op0);
2050 }
2051 if ((Subtarget->hasNEON())) {
2052 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv8i16, RC: &ARM::QPRRegClass, Op0);
2053 }
2054 return Register();
2055}
2056
2057Register fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, Register Op0) {
2058 if (RetVT.SimpleTy != MVT::v2i32)
2059 return Register();
2060 if ((Subtarget->hasNEON())) {
2061 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv2i32, RC: &ARM::DPRRegClass, Op0);
2062 }
2063 return Register();
2064}
2065
2066Register fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, Register Op0) {
2067 if (RetVT.SimpleTy != MVT::v4i32)
2068 return Register();
2069 if ((Subtarget->hasMVEIntegerOps())) {
2070 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLZs32, RC: &ARM::MQPRRegClass, Op0);
2071 }
2072 if ((Subtarget->hasNEON())) {
2073 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv4i32, RC: &ARM::QPRRegClass, Op0);
2074 }
2075 return Register();
2076}
2077
2078Register fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, Register Op0) {
2079 switch (VT.SimpleTy) {
2080 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
2081 case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0);
2082 case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0);
2083 case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0);
2084 case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0);
2085 case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0);
2086 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
2087 default: return Register();
2088 }
2089}
2090
2091// FastEmit functions for ISD::CTPOP.
2092
2093Register fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, Register Op0) {
2094 if (RetVT.SimpleTy != MVT::v8i8)
2095 return Register();
2096 if ((Subtarget->hasNEON())) {
2097 return fastEmitInst_r(MachineInstOpcode: ARM::VCNTd, RC: &ARM::DPRRegClass, Op0);
2098 }
2099 return Register();
2100}
2101
2102Register fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, Register Op0) {
2103 if (RetVT.SimpleTy != MVT::v16i8)
2104 return Register();
2105 if ((Subtarget->hasNEON())) {
2106 return fastEmitInst_r(MachineInstOpcode: ARM::VCNTq, RC: &ARM::QPRRegClass, Op0);
2107 }
2108 return Register();
2109}
2110
2111Register fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, Register Op0) {
2112 switch (VT.SimpleTy) {
2113 case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0);
2114 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
2115 default: return Register();
2116 }
2117}
2118
2119// FastEmit functions for ISD::FABS.
2120
2121Register fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, Register Op0) {
2122 if (RetVT.SimpleTy != MVT::f16)
2123 return Register();
2124 if ((Subtarget->hasFullFP16())) {
2125 return fastEmitInst_r(MachineInstOpcode: ARM::VABSH, RC: &ARM::HPRRegClass, Op0);
2126 }
2127 return Register();
2128}
2129
2130Register fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, Register Op0) {
2131 if (RetVT.SimpleTy != MVT::f32)
2132 return Register();
2133 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
2134 return fastEmitInst_r(MachineInstOpcode: ARM::VABSS, RC: &ARM::SPRRegClass, Op0);
2135 }
2136 return Register();
2137}
2138
2139Register fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, Register Op0) {
2140 if (RetVT.SimpleTy != MVT::f64)
2141 return Register();
2142 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2143 return fastEmitInst_r(MachineInstOpcode: ARM::VABSD, RC: &ARM::DPRRegClass, Op0);
2144 }
2145 return Register();
2146}
2147
2148Register fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, Register Op0) {
2149 if (RetVT.SimpleTy != MVT::v4f16)
2150 return Register();
2151 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2152 return fastEmitInst_r(MachineInstOpcode: ARM::VABShd, RC: &ARM::DPRRegClass, Op0);
2153 }
2154 return Register();
2155}
2156
2157Register fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, Register Op0) {
2158 if (RetVT.SimpleTy != MVT::v8f16)
2159 return Register();
2160 if ((Subtarget->hasMVEIntegerOps())) {
2161 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSf16, RC: &ARM::MQPRRegClass, Op0);
2162 }
2163 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2164 return fastEmitInst_r(MachineInstOpcode: ARM::VABShq, RC: &ARM::QPRRegClass, Op0);
2165 }
2166 return Register();
2167}
2168
2169Register fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, Register Op0) {
2170 if (RetVT.SimpleTy != MVT::v2f32)
2171 return Register();
2172 if ((Subtarget->hasNEON())) {
2173 return fastEmitInst_r(MachineInstOpcode: ARM::VABSfd, RC: &ARM::DPRRegClass, Op0);
2174 }
2175 return Register();
2176}
2177
2178Register fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, Register Op0) {
2179 if (RetVT.SimpleTy != MVT::v4f32)
2180 return Register();
2181 if ((Subtarget->hasMVEIntegerOps())) {
2182 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSf32, RC: &ARM::MQPRRegClass, Op0);
2183 }
2184 if ((Subtarget->hasNEON())) {
2185 return fastEmitInst_r(MachineInstOpcode: ARM::VABSfq, RC: &ARM::QPRRegClass, Op0);
2186 }
2187 return Register();
2188}
2189
2190Register fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, Register Op0) {
2191 switch (VT.SimpleTy) {
2192 case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0);
2193 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
2194 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
2195 case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0);
2196 case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0);
2197 case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0);
2198 case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0);
2199 default: return Register();
2200 }
2201}
2202
2203// FastEmit functions for ISD::FCEIL.
2204
2205Register fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
2206 if (RetVT.SimpleTy != MVT::f16)
2207 return Register();
2208 if ((Subtarget->hasFullFP16())) {
2209 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPH, RC: &ARM::HPRRegClass, Op0);
2210 }
2211 return Register();
2212}
2213
2214Register fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
2215 if (RetVT.SimpleTy != MVT::f32)
2216 return Register();
2217 if ((Subtarget->hasFPARMv8Base())) {
2218 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPS, RC: &ARM::SPRRegClass, Op0);
2219 }
2220 return Register();
2221}
2222
2223Register fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
2224 if (RetVT.SimpleTy != MVT::f64)
2225 return Register();
2226 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2227 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPD, RC: &ARM::DPRRegClass, Op0);
2228 }
2229 return Register();
2230}
2231
2232Register fastEmit_ISD_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
2233 if (RetVT.SimpleTy != MVT::v4f16)
2234 return Register();
2235 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2236 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNDh, RC: &ARM::DPRRegClass, Op0);
2237 }
2238 return Register();
2239}
2240
2241Register fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
2242 if (RetVT.SimpleTy != MVT::v8f16)
2243 return Register();
2244 if ((Subtarget->hasMVEFloatOps())) {
2245 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16P, RC: &ARM::MQPRRegClass, Op0);
2246 }
2247 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2248 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNQh, RC: &ARM::QPRRegClass, Op0);
2249 }
2250 return Register();
2251}
2252
2253Register fastEmit_ISD_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
2254 if (RetVT.SimpleTy != MVT::v2f32)
2255 return Register();
2256 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2257 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNDf, RC: &ARM::DPRRegClass, Op0);
2258 }
2259 return Register();
2260}
2261
2262Register fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
2263 if (RetVT.SimpleTy != MVT::v4f32)
2264 return Register();
2265 if ((Subtarget->hasMVEFloatOps())) {
2266 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32P, RC: &ARM::MQPRRegClass, Op0);
2267 }
2268 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2269 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNQf, RC: &ARM::QPRRegClass, Op0);
2270 }
2271 return Register();
2272}
2273
2274Register fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
2275 switch (VT.SimpleTy) {
2276 case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0);
2277 case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0);
2278 case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0);
2279 case MVT::v4f16: return fastEmit_ISD_FCEIL_MVT_v4f16_r(RetVT, Op0);
2280 case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0);
2281 case MVT::v2f32: return fastEmit_ISD_FCEIL_MVT_v2f32_r(RetVT, Op0);
2282 case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0);
2283 default: return Register();
2284 }
2285}
2286
2287// FastEmit functions for ISD::FFLOOR.
2288
2289Register fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
2290 if (RetVT.SimpleTy != MVT::f16)
2291 return Register();
2292 if ((Subtarget->hasFullFP16())) {
2293 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMH, RC: &ARM::HPRRegClass, Op0);
2294 }
2295 return Register();
2296}
2297
2298Register fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
2299 if (RetVT.SimpleTy != MVT::f32)
2300 return Register();
2301 if ((Subtarget->hasFPARMv8Base())) {
2302 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMS, RC: &ARM::SPRRegClass, Op0);
2303 }
2304 return Register();
2305}
2306
2307Register fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
2308 if (RetVT.SimpleTy != MVT::f64)
2309 return Register();
2310 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2311 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMD, RC: &ARM::DPRRegClass, Op0);
2312 }
2313 return Register();
2314}
2315
2316Register fastEmit_ISD_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
2317 if (RetVT.SimpleTy != MVT::v4f16)
2318 return Register();
2319 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2320 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNDh, RC: &ARM::DPRRegClass, Op0);
2321 }
2322 return Register();
2323}
2324
2325Register fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
2326 if (RetVT.SimpleTy != MVT::v8f16)
2327 return Register();
2328 if ((Subtarget->hasMVEFloatOps())) {
2329 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16M, RC: &ARM::MQPRRegClass, Op0);
2330 }
2331 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2332 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNQh, RC: &ARM::QPRRegClass, Op0);
2333 }
2334 return Register();
2335}
2336
2337Register fastEmit_ISD_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
2338 if (RetVT.SimpleTy != MVT::v2f32)
2339 return Register();
2340 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2341 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNDf, RC: &ARM::DPRRegClass, Op0);
2342 }
2343 return Register();
2344}
2345
2346Register fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
2347 if (RetVT.SimpleTy != MVT::v4f32)
2348 return Register();
2349 if ((Subtarget->hasMVEFloatOps())) {
2350 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32M, RC: &ARM::MQPRRegClass, Op0);
2351 }
2352 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2353 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNQf, RC: &ARM::QPRRegClass, Op0);
2354 }
2355 return Register();
2356}
2357
2358Register fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
2359 switch (VT.SimpleTy) {
2360 case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0);
2361 case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0);
2362 case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0);
2363 case MVT::v4f16: return fastEmit_ISD_FFLOOR_MVT_v4f16_r(RetVT, Op0);
2364 case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0);
2365 case MVT::v2f32: return fastEmit_ISD_FFLOOR_MVT_v2f32_r(RetVT, Op0);
2366 case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0);
2367 default: return Register();
2368 }
2369}
2370
2371// FastEmit functions for ISD::FNEARBYINT.
2372
2373Register fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
2374 if (RetVT.SimpleTy != MVT::f16)
2375 return Register();
2376 if ((Subtarget->hasFullFP16())) {
2377 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRH, RC: &ARM::HPRRegClass, Op0);
2378 }
2379 return Register();
2380}
2381
2382Register fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
2383 if (RetVT.SimpleTy != MVT::f32)
2384 return Register();
2385 if ((Subtarget->hasFPARMv8Base())) {
2386 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRS, RC: &ARM::SPRRegClass, Op0);
2387 }
2388 return Register();
2389}
2390
2391Register fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
2392 if (RetVT.SimpleTy != MVT::f64)
2393 return Register();
2394 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2395 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRD, RC: &ARM::DPRRegClass, Op0);
2396 }
2397 return Register();
2398}
2399
2400Register fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
2401 switch (VT.SimpleTy) {
2402 case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0);
2403 case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0);
2404 case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0);
2405 default: return Register();
2406 }
2407}
2408
2409// FastEmit functions for ISD::FNEG.
2410
2411Register fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, Register Op0) {
2412 if (RetVT.SimpleTy != MVT::f16)
2413 return Register();
2414 if ((Subtarget->hasFullFP16())) {
2415 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGH, RC: &ARM::HPRRegClass, Op0);
2416 }
2417 return Register();
2418}
2419
2420Register fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, Register Op0) {
2421 if (RetVT.SimpleTy != MVT::f32)
2422 return Register();
2423 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
2424 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGS, RC: &ARM::SPRRegClass, Op0);
2425 }
2426 return Register();
2427}
2428
2429Register fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, Register Op0) {
2430 if (RetVT.SimpleTy != MVT::f64)
2431 return Register();
2432 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2433 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGD, RC: &ARM::DPRRegClass, Op0);
2434 }
2435 return Register();
2436}
2437
2438Register fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, Register Op0) {
2439 if (RetVT.SimpleTy != MVT::v4f16)
2440 return Register();
2441 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2442 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGhd, RC: &ARM::DPRRegClass, Op0);
2443 }
2444 return Register();
2445}
2446
2447Register fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, Register Op0) {
2448 if (RetVT.SimpleTy != MVT::v8f16)
2449 return Register();
2450 if ((Subtarget->hasMVEIntegerOps())) {
2451 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VNEGf16, RC: &ARM::MQPRRegClass, Op0);
2452 }
2453 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2454 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGhq, RC: &ARM::QPRRegClass, Op0);
2455 }
2456 return Register();
2457}
2458
2459Register fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, Register Op0) {
2460 if (RetVT.SimpleTy != MVT::v2f32)
2461 return Register();
2462 if ((Subtarget->hasNEON())) {
2463 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGfd, RC: &ARM::DPRRegClass, Op0);
2464 }
2465 return Register();
2466}
2467
2468Register fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, Register Op0) {
2469 if (RetVT.SimpleTy != MVT::v4f32)
2470 return Register();
2471 if ((Subtarget->hasMVEIntegerOps())) {
2472 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VNEGf32, RC: &ARM::MQPRRegClass, Op0);
2473 }
2474 if ((Subtarget->hasNEON())) {
2475 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGf32q, RC: &ARM::QPRRegClass, Op0);
2476 }
2477 return Register();
2478}
2479
2480Register fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, Register Op0) {
2481 switch (VT.SimpleTy) {
2482 case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0);
2483 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
2484 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
2485 case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0);
2486 case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0);
2487 case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0);
2488 case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0);
2489 default: return Register();
2490 }
2491}
2492
2493// FastEmit functions for ISD::FP_EXTEND.
2494
2495Register fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
2496 if (RetVT.SimpleTy != MVT::f64)
2497 return Register();
2498 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2499 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTDS, RC: &ARM::DPRRegClass, Op0);
2500 }
2501 return Register();
2502}
2503
2504Register fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
2505 if (RetVT.SimpleTy != MVT::v4f32)
2506 return Register();
2507 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2f, RC: &ARM::QPRRegClass, Op0);
2508}
2509
2510Register fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
2511 switch (VT.SimpleTy) {
2512 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
2513 case MVT::v4f16: return fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
2514 default: return Register();
2515 }
2516}
2517
2518// FastEmit functions for ISD::FP_ROUND.
2519
2520Register fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
2521 if (RetVT.SimpleTy != MVT::f32)
2522 return Register();
2523 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2524 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTSD, RC: &ARM::SPRRegClass, Op0);
2525 }
2526 return Register();
2527}
2528
2529Register fastEmit_ISD_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
2530 if (RetVT.SimpleTy != MVT::v4f16)
2531 return Register();
2532 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2h, RC: &ARM::DPRRegClass, Op0);
2533}
2534
2535Register fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
2536 switch (VT.SimpleTy) {
2537 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
2538 case MVT::v4f32: return fastEmit_ISD_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
2539 default: return Register();
2540 }
2541}
2542
2543// FastEmit functions for ISD::FP_TO_SINT.
2544
2545Register fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2546 if (RetVT.SimpleTy != MVT::v4i16)
2547 return Register();
2548 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2549 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2sd, RC: &ARM::DPRRegClass, Op0);
2550 }
2551 return Register();
2552}
2553
2554Register fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2555 if (RetVT.SimpleTy != MVT::v8i16)
2556 return Register();
2557 if ((Subtarget->hasMVEFloatOps())) {
2558 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTs16f16z, RC: &ARM::MQPRRegClass, Op0);
2559 }
2560 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2561 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2sq, RC: &ARM::QPRRegClass, Op0);
2562 }
2563 return Register();
2564}
2565
2566Register fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2567 if (RetVT.SimpleTy != MVT::v2i32)
2568 return Register();
2569 if ((Subtarget->hasNEON())) {
2570 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2sd, RC: &ARM::DPRRegClass, Op0);
2571 }
2572 return Register();
2573}
2574
2575Register fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2576 if (RetVT.SimpleTy != MVT::v4i32)
2577 return Register();
2578 if ((Subtarget->hasMVEFloatOps())) {
2579 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTs32f32z, RC: &ARM::MQPRRegClass, Op0);
2580 }
2581 if ((Subtarget->hasNEON())) {
2582 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2sq, RC: &ARM::QPRRegClass, Op0);
2583 }
2584 return Register();
2585}
2586
2587Register fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
2588 switch (VT.SimpleTy) {
2589 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
2590 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
2591 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
2592 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
2593 default: return Register();
2594 }
2595}
2596
2597// FastEmit functions for ISD::FP_TO_UINT.
2598
2599Register fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2600 if (RetVT.SimpleTy != MVT::v4i16)
2601 return Register();
2602 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2603 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2ud, RC: &ARM::DPRRegClass, Op0);
2604 }
2605 return Register();
2606}
2607
2608Register fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2609 if (RetVT.SimpleTy != MVT::v8i16)
2610 return Register();
2611 if ((Subtarget->hasMVEFloatOps())) {
2612 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTu16f16z, RC: &ARM::MQPRRegClass, Op0);
2613 }
2614 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2615 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2uq, RC: &ARM::QPRRegClass, Op0);
2616 }
2617 return Register();
2618}
2619
2620Register fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2621 if (RetVT.SimpleTy != MVT::v2i32)
2622 return Register();
2623 if ((Subtarget->hasNEON())) {
2624 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2ud, RC: &ARM::DPRRegClass, Op0);
2625 }
2626 return Register();
2627}
2628
2629Register fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2630 if (RetVT.SimpleTy != MVT::v4i32)
2631 return Register();
2632 if ((Subtarget->hasMVEFloatOps())) {
2633 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTu32f32z, RC: &ARM::MQPRRegClass, Op0);
2634 }
2635 if ((Subtarget->hasNEON())) {
2636 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2uq, RC: &ARM::QPRRegClass, Op0);
2637 }
2638 return Register();
2639}
2640
2641Register fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
2642 switch (VT.SimpleTy) {
2643 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
2644 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
2645 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
2646 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
2647 default: return Register();
2648 }
2649}
2650
2651// FastEmit functions for ISD::FRINT.
2652
2653Register fastEmit_ISD_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
2654 if (RetVT.SimpleTy != MVT::f16)
2655 return Register();
2656 if ((Subtarget->hasFullFP16())) {
2657 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXH, RC: &ARM::HPRRegClass, Op0);
2658 }
2659 return Register();
2660}
2661
2662Register fastEmit_ISD_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
2663 if (RetVT.SimpleTy != MVT::f32)
2664 return Register();
2665 if ((Subtarget->hasFPARMv8Base())) {
2666 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXS, RC: &ARM::SPRRegClass, Op0);
2667 }
2668 return Register();
2669}
2670
2671Register fastEmit_ISD_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
2672 if (RetVT.SimpleTy != MVT::f64)
2673 return Register();
2674 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2675 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXD, RC: &ARM::DPRRegClass, Op0);
2676 }
2677 return Register();
2678}
2679
2680Register fastEmit_ISD_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2681 if (RetVT.SimpleTy != MVT::v4f16)
2682 return Register();
2683 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2684 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNDh, RC: &ARM::DPRRegClass, Op0);
2685 }
2686 return Register();
2687}
2688
2689Register fastEmit_ISD_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2690 if (RetVT.SimpleTy != MVT::v8f16)
2691 return Register();
2692 if ((Subtarget->hasMVEFloatOps())) {
2693 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16X, RC: &ARM::MQPRRegClass, Op0);
2694 }
2695 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2696 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNQh, RC: &ARM::QPRRegClass, Op0);
2697 }
2698 return Register();
2699}
2700
2701Register fastEmit_ISD_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2702 if (RetVT.SimpleTy != MVT::v2f32)
2703 return Register();
2704 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2705 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNDf, RC: &ARM::DPRRegClass, Op0);
2706 }
2707 return Register();
2708}
2709
2710Register fastEmit_ISD_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2711 if (RetVT.SimpleTy != MVT::v4f32)
2712 return Register();
2713 if ((Subtarget->hasMVEFloatOps())) {
2714 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32X, RC: &ARM::MQPRRegClass, Op0);
2715 }
2716 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2717 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNQf, RC: &ARM::QPRRegClass, Op0);
2718 }
2719 return Register();
2720}
2721
2722Register fastEmit_ISD_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
2723 switch (VT.SimpleTy) {
2724 case MVT::f16: return fastEmit_ISD_FRINT_MVT_f16_r(RetVT, Op0);
2725 case MVT::f32: return fastEmit_ISD_FRINT_MVT_f32_r(RetVT, Op0);
2726 case MVT::f64: return fastEmit_ISD_FRINT_MVT_f64_r(RetVT, Op0);
2727 case MVT::v4f16: return fastEmit_ISD_FRINT_MVT_v4f16_r(RetVT, Op0);
2728 case MVT::v8f16: return fastEmit_ISD_FRINT_MVT_v8f16_r(RetVT, Op0);
2729 case MVT::v2f32: return fastEmit_ISD_FRINT_MVT_v2f32_r(RetVT, Op0);
2730 case MVT::v4f32: return fastEmit_ISD_FRINT_MVT_v4f32_r(RetVT, Op0);
2731 default: return Register();
2732 }
2733}
2734
2735// FastEmit functions for ISD::FROUND.
2736
2737Register fastEmit_ISD_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
2738 if (RetVT.SimpleTy != MVT::f16)
2739 return Register();
2740 if ((Subtarget->hasFullFP16())) {
2741 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAH, RC: &ARM::HPRRegClass, Op0);
2742 }
2743 return Register();
2744}
2745
2746Register fastEmit_ISD_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
2747 if (RetVT.SimpleTy != MVT::f32)
2748 return Register();
2749 if ((Subtarget->hasFPARMv8Base())) {
2750 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAS, RC: &ARM::SPRRegClass, Op0);
2751 }
2752 return Register();
2753}
2754
2755Register fastEmit_ISD_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
2756 if (RetVT.SimpleTy != MVT::f64)
2757 return Register();
2758 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2759 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAD, RC: &ARM::DPRRegClass, Op0);
2760 }
2761 return Register();
2762}
2763
2764Register fastEmit_ISD_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
2765 if (RetVT.SimpleTy != MVT::v4f16)
2766 return Register();
2767 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2768 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANDh, RC: &ARM::DPRRegClass, Op0);
2769 }
2770 return Register();
2771}
2772
2773Register fastEmit_ISD_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
2774 if (RetVT.SimpleTy != MVT::v8f16)
2775 return Register();
2776 if ((Subtarget->hasMVEFloatOps())) {
2777 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16A, RC: &ARM::MQPRRegClass, Op0);
2778 }
2779 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2780 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANQh, RC: &ARM::QPRRegClass, Op0);
2781 }
2782 return Register();
2783}
2784
2785Register fastEmit_ISD_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
2786 if (RetVT.SimpleTy != MVT::v2f32)
2787 return Register();
2788 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2789 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANDf, RC: &ARM::DPRRegClass, Op0);
2790 }
2791 return Register();
2792}
2793
2794Register fastEmit_ISD_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
2795 if (RetVT.SimpleTy != MVT::v4f32)
2796 return Register();
2797 if ((Subtarget->hasMVEFloatOps())) {
2798 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32A, RC: &ARM::MQPRRegClass, Op0);
2799 }
2800 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2801 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANQf, RC: &ARM::QPRRegClass, Op0);
2802 }
2803 return Register();
2804}
2805
2806Register fastEmit_ISD_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
2807 switch (VT.SimpleTy) {
2808 case MVT::f16: return fastEmit_ISD_FROUND_MVT_f16_r(RetVT, Op0);
2809 case MVT::f32: return fastEmit_ISD_FROUND_MVT_f32_r(RetVT, Op0);
2810 case MVT::f64: return fastEmit_ISD_FROUND_MVT_f64_r(RetVT, Op0);
2811 case MVT::v4f16: return fastEmit_ISD_FROUND_MVT_v4f16_r(RetVT, Op0);
2812 case MVT::v8f16: return fastEmit_ISD_FROUND_MVT_v8f16_r(RetVT, Op0);
2813 case MVT::v2f32: return fastEmit_ISD_FROUND_MVT_v2f32_r(RetVT, Op0);
2814 case MVT::v4f32: return fastEmit_ISD_FROUND_MVT_v4f32_r(RetVT, Op0);
2815 default: return Register();
2816 }
2817}
2818
2819// FastEmit functions for ISD::FROUNDEVEN.
2820
2821Register fastEmit_ISD_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
2822 if (RetVT.SimpleTy != MVT::f16)
2823 return Register();
2824 if ((Subtarget->hasFullFP16())) {
2825 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNH, RC: &ARM::HPRRegClass, Op0);
2826 }
2827 return Register();
2828}
2829
2830Register fastEmit_ISD_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
2831 if (RetVT.SimpleTy != MVT::f32)
2832 return Register();
2833 if ((Subtarget->hasFPARMv8Base())) {
2834 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNS, RC: &ARM::SPRRegClass, Op0);
2835 }
2836 return Register();
2837}
2838
2839Register fastEmit_ISD_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
2840 if (RetVT.SimpleTy != MVT::f64)
2841 return Register();
2842 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2843 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTND, RC: &ARM::DPRRegClass, Op0);
2844 }
2845 return Register();
2846}
2847
2848Register fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
2849 if (RetVT.SimpleTy != MVT::v8f16)
2850 return Register();
2851 if ((Subtarget->hasMVEFloatOps())) {
2852 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16N, RC: &ARM::MQPRRegClass, Op0);
2853 }
2854 return Register();
2855}
2856
2857Register fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
2858 if (RetVT.SimpleTy != MVT::v4f32)
2859 return Register();
2860 if ((Subtarget->hasMVEFloatOps())) {
2861 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32N, RC: &ARM::MQPRRegClass, Op0);
2862 }
2863 return Register();
2864}
2865
2866Register fastEmit_ISD_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
2867 switch (VT.SimpleTy) {
2868 case MVT::f16: return fastEmit_ISD_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
2869 case MVT::f32: return fastEmit_ISD_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
2870 case MVT::f64: return fastEmit_ISD_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
2871 case MVT::v8f16: return fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
2872 case MVT::v4f32: return fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
2873 default: return Register();
2874 }
2875}
2876
2877// FastEmit functions for ISD::FSQRT.
2878
2879Register fastEmit_ISD_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
2880 if (RetVT.SimpleTy != MVT::f16)
2881 return Register();
2882 if ((Subtarget->hasFullFP16())) {
2883 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTH, RC: &ARM::HPRRegClass, Op0);
2884 }
2885 return Register();
2886}
2887
2888Register fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
2889 if (RetVT.SimpleTy != MVT::f32)
2890 return Register();
2891 if ((Subtarget->hasVFP2Base())) {
2892 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTS, RC: &ARM::SPRRegClass, Op0);
2893 }
2894 return Register();
2895}
2896
2897Register fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
2898 if (RetVT.SimpleTy != MVT::f64)
2899 return Register();
2900 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2901 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTD, RC: &ARM::DPRRegClass, Op0);
2902 }
2903 return Register();
2904}
2905
2906Register fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
2907 switch (VT.SimpleTy) {
2908 case MVT::f16: return fastEmit_ISD_FSQRT_MVT_f16_r(RetVT, Op0);
2909 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
2910 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
2911 default: return Register();
2912 }
2913}
2914
2915// FastEmit functions for ISD::FTRUNC.
2916
2917Register fastEmit_ISD_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
2918 if (RetVT.SimpleTy != MVT::f16)
2919 return Register();
2920 if ((Subtarget->hasFullFP16())) {
2921 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZH, RC: &ARM::HPRRegClass, Op0);
2922 }
2923 return Register();
2924}
2925
2926Register fastEmit_ISD_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
2927 if (RetVT.SimpleTy != MVT::f32)
2928 return Register();
2929 if ((Subtarget->hasFPARMv8Base())) {
2930 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZS, RC: &ARM::SPRRegClass, Op0);
2931 }
2932 return Register();
2933}
2934
2935Register fastEmit_ISD_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
2936 if (RetVT.SimpleTy != MVT::f64)
2937 return Register();
2938 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2939 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZD, RC: &ARM::DPRRegClass, Op0);
2940 }
2941 return Register();
2942}
2943
2944Register fastEmit_ISD_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
2945 if (RetVT.SimpleTy != MVT::v4f16)
2946 return Register();
2947 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2948 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNDh, RC: &ARM::DPRRegClass, Op0);
2949 }
2950 return Register();
2951}
2952
2953Register fastEmit_ISD_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
2954 if (RetVT.SimpleTy != MVT::v8f16)
2955 return Register();
2956 if ((Subtarget->hasMVEFloatOps())) {
2957 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16Z, RC: &ARM::MQPRRegClass, Op0);
2958 }
2959 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2960 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNQh, RC: &ARM::QPRRegClass, Op0);
2961 }
2962 return Register();
2963}
2964
2965Register fastEmit_ISD_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
2966 if (RetVT.SimpleTy != MVT::v2f32)
2967 return Register();
2968 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2969 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNDf, RC: &ARM::DPRRegClass, Op0);
2970 }
2971 return Register();
2972}
2973
2974Register fastEmit_ISD_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
2975 if (RetVT.SimpleTy != MVT::v4f32)
2976 return Register();
2977 if ((Subtarget->hasMVEFloatOps())) {
2978 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32Z, RC: &ARM::MQPRRegClass, Op0);
2979 }
2980 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2981 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNQf, RC: &ARM::QPRRegClass, Op0);
2982 }
2983 return Register();
2984}
2985
2986Register fastEmit_ISD_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
2987 switch (VT.SimpleTy) {
2988 case MVT::f16: return fastEmit_ISD_FTRUNC_MVT_f16_r(RetVT, Op0);
2989 case MVT::f32: return fastEmit_ISD_FTRUNC_MVT_f32_r(RetVT, Op0);
2990 case MVT::f64: return fastEmit_ISD_FTRUNC_MVT_f64_r(RetVT, Op0);
2991 case MVT::v4f16: return fastEmit_ISD_FTRUNC_MVT_v4f16_r(RetVT, Op0);
2992 case MVT::v8f16: return fastEmit_ISD_FTRUNC_MVT_v8f16_r(RetVT, Op0);
2993 case MVT::v2f32: return fastEmit_ISD_FTRUNC_MVT_v2f32_r(RetVT, Op0);
2994 case MVT::v4f32: return fastEmit_ISD_FTRUNC_MVT_v4f32_r(RetVT, Op0);
2995 default: return Register();
2996 }
2997}
2998
2999// FastEmit functions for ISD::SET_FPENV.
3000
3001Register fastEmit_ISD_SET_FPENV_MVT_i32_r(MVT RetVT, Register Op0) {
3002 if (RetVT.SimpleTy != MVT::isVoid)
3003 return Register();
3004 return fastEmitInst_r(MachineInstOpcode: ARM::VMSR, RC: &ARM::GPRnopcRegClass, Op0);
3005}
3006
3007Register fastEmit_ISD_SET_FPENV_r(MVT VT, MVT RetVT, Register Op0) {
3008 switch (VT.SimpleTy) {
3009 case MVT::i32: return fastEmit_ISD_SET_FPENV_MVT_i32_r(RetVT, Op0);
3010 default: return Register();
3011 }
3012}
3013
3014// FastEmit functions for ISD::SIGN_EXTEND.
3015
3016Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i8_r(MVT RetVT, Register Op0) {
3017 if (RetVT.SimpleTy != MVT::v8i16)
3018 return Register();
3019 if ((Subtarget->hasNEON())) {
3020 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLsv8i16, RC: &ARM::QPRRegClass, Op0);
3021 }
3022 return Register();
3023}
3024
3025Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i16_r(MVT RetVT, Register Op0) {
3026 if (RetVT.SimpleTy != MVT::v4i32)
3027 return Register();
3028 if ((Subtarget->hasNEON())) {
3029 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLsv4i32, RC: &ARM::QPRRegClass, Op0);
3030 }
3031 return Register();
3032}
3033
3034Register fastEmit_ISD_SIGN_EXTEND_MVT_v2i32_r(MVT RetVT, Register Op0) {
3035 if (RetVT.SimpleTy != MVT::v2i64)
3036 return Register();
3037 if ((Subtarget->hasNEON())) {
3038 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLsv2i64, RC: &ARM::QPRRegClass, Op0);
3039 }
3040 return Register();
3041}
3042
3043Register fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3044 switch (VT.SimpleTy) {
3045 case MVT::v8i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i8_r(RetVT, Op0);
3046 case MVT::v4i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i16_r(RetVT, Op0);
3047 case MVT::v2i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i32_r(RetVT, Op0);
3048 default: return Register();
3049 }
3050}
3051
3052// FastEmit functions for ISD::SINT_TO_FP.
3053
3054Register fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
3055 if (RetVT.SimpleTy != MVT::v4f16)
3056 return Register();
3057 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3058 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2hd, RC: &ARM::DPRRegClass, Op0);
3059 }
3060 return Register();
3061}
3062
3063Register fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
3064 if (RetVT.SimpleTy != MVT::v8f16)
3065 return Register();
3066 if ((Subtarget->hasMVEFloatOps())) {
3067 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf16s16n, RC: &ARM::MQPRRegClass, Op0);
3068 }
3069 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3070 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2hq, RC: &ARM::QPRRegClass, Op0);
3071 }
3072 return Register();
3073}
3074
3075Register fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
3076 if (RetVT.SimpleTy != MVT::v2f32)
3077 return Register();
3078 if ((Subtarget->hasNEON())) {
3079 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2fd, RC: &ARM::DPRRegClass, Op0);
3080 }
3081 return Register();
3082}
3083
3084Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
3085 if (RetVT.SimpleTy != MVT::v4f32)
3086 return Register();
3087 if ((Subtarget->hasMVEFloatOps())) {
3088 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf32s32n, RC: &ARM::MQPRRegClass, Op0);
3089 }
3090 if ((Subtarget->hasNEON())) {
3091 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2fq, RC: &ARM::QPRRegClass, Op0);
3092 }
3093 return Register();
3094}
3095
3096Register fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3097 switch (VT.SimpleTy) {
3098 case MVT::v4i16: return fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
3099 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3100 case MVT::v2i32: return fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
3101 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3102 default: return Register();
3103 }
3104}
3105
3106// FastEmit functions for ISD::TRUNCATE.
3107
3108Register fastEmit_ISD_TRUNCATE_MVT_v8i16_r(MVT RetVT, Register Op0) {
3109 if (RetVT.SimpleTy != MVT::v8i8)
3110 return Register();
3111 if ((Subtarget->hasNEON())) {
3112 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVNv8i8, RC: &ARM::DPRRegClass, Op0);
3113 }
3114 return Register();
3115}
3116
3117Register fastEmit_ISD_TRUNCATE_MVT_v4i32_r(MVT RetVT, Register Op0) {
3118 if (RetVT.SimpleTy != MVT::v4i16)
3119 return Register();
3120 if ((Subtarget->hasNEON())) {
3121 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVNv4i16, RC: &ARM::DPRRegClass, Op0);
3122 }
3123 return Register();
3124}
3125
3126Register fastEmit_ISD_TRUNCATE_MVT_v2i64_r(MVT RetVT, Register Op0) {
3127 if (RetVT.SimpleTy != MVT::v2i32)
3128 return Register();
3129 if ((Subtarget->hasNEON())) {
3130 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVNv2i32, RC: &ARM::DPRRegClass, Op0);
3131 }
3132 return Register();
3133}
3134
3135Register fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, Register Op0) {
3136 switch (VT.SimpleTy) {
3137 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i16_r(RetVT, Op0);
3138 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_MVT_v4i32_r(RetVT, Op0);
3139 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_MVT_v2i64_r(RetVT, Op0);
3140 default: return Register();
3141 }
3142}
3143
3144// FastEmit functions for ISD::UINT_TO_FP.
3145
3146Register fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
3147 if (RetVT.SimpleTy != MVT::v4f16)
3148 return Register();
3149 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3150 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2hd, RC: &ARM::DPRRegClass, Op0);
3151 }
3152 return Register();
3153}
3154
3155Register fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
3156 if (RetVT.SimpleTy != MVT::v8f16)
3157 return Register();
3158 if ((Subtarget->hasMVEFloatOps())) {
3159 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf16u16n, RC: &ARM::MQPRRegClass, Op0);
3160 }
3161 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3162 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2hq, RC: &ARM::QPRRegClass, Op0);
3163 }
3164 return Register();
3165}
3166
3167Register fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
3168 if (RetVT.SimpleTy != MVT::v2f32)
3169 return Register();
3170 if ((Subtarget->hasNEON())) {
3171 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2fd, RC: &ARM::DPRRegClass, Op0);
3172 }
3173 return Register();
3174}
3175
3176Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
3177 if (RetVT.SimpleTy != MVT::v4f32)
3178 return Register();
3179 if ((Subtarget->hasMVEFloatOps())) {
3180 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf32u32n, RC: &ARM::MQPRRegClass, Op0);
3181 }
3182 if ((Subtarget->hasNEON())) {
3183 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2fq, RC: &ARM::QPRRegClass, Op0);
3184 }
3185 return Register();
3186}
3187
3188Register fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3189 switch (VT.SimpleTy) {
3190 case MVT::v4i16: return fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
3191 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3192 case MVT::v2i32: return fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
3193 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3194 default: return Register();
3195 }
3196}
3197
3198// FastEmit functions for ISD::VECREDUCE_ADD.
3199
3200Register fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(MVT RetVT, Register Op0) {
3201 if (RetVT.SimpleTy != MVT::i32)
3202 return Register();
3203 if ((Subtarget->hasMVEIntegerOps())) {
3204 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu8no_acc, RC: &ARM::tGPREvenRegClass, Op0);
3205 }
3206 return Register();
3207}
3208
3209Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(MVT RetVT, Register Op0) {
3210 if (RetVT.SimpleTy != MVT::i32)
3211 return Register();
3212 if ((Subtarget->hasMVEIntegerOps())) {
3213 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu16no_acc, RC: &ARM::tGPREvenRegClass, Op0);
3214 }
3215 return Register();
3216}
3217
3218Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(MVT RetVT, Register Op0) {
3219 if (RetVT.SimpleTy != MVT::i32)
3220 return Register();
3221 if ((Subtarget->hasMVEIntegerOps())) {
3222 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu32no_acc, RC: &ARM::tGPREvenRegClass, Op0);
3223 }
3224 return Register();
3225}
3226
3227Register fastEmit_ISD_VECREDUCE_ADD_r(MVT VT, MVT RetVT, Register Op0) {
3228 switch (VT.SimpleTy) {
3229 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(RetVT, Op0);
3230 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(RetVT, Op0);
3231 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(RetVT, Op0);
3232 default: return Register();
3233 }
3234}
3235
3236// FastEmit functions for ISD::ZERO_EXTEND.
3237
3238Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i8_r(MVT RetVT, Register Op0) {
3239 if (RetVT.SimpleTy != MVT::v8i16)
3240 return Register();
3241 if ((Subtarget->hasNEON())) {
3242 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv8i16, RC: &ARM::QPRRegClass, Op0);
3243 }
3244 return Register();
3245}
3246
3247Register fastEmit_ISD_ZERO_EXTEND_MVT_v4i16_r(MVT RetVT, Register Op0) {
3248 if (RetVT.SimpleTy != MVT::v4i32)
3249 return Register();
3250 if ((Subtarget->hasNEON())) {
3251 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv4i32, RC: &ARM::QPRRegClass, Op0);
3252 }
3253 return Register();
3254}
3255
3256Register fastEmit_ISD_ZERO_EXTEND_MVT_v2i32_r(MVT RetVT, Register Op0) {
3257 if (RetVT.SimpleTy != MVT::v2i64)
3258 return Register();
3259 if ((Subtarget->hasNEON())) {
3260 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv2i64, RC: &ARM::QPRRegClass, Op0);
3261 }
3262 return Register();
3263}
3264
3265Register fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3266 switch (VT.SimpleTy) {
3267 case MVT::v8i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i8_r(RetVT, Op0);
3268 case MVT::v4i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v4i16_r(RetVT, Op0);
3269 case MVT::v2i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v2i32_r(RetVT, Op0);
3270 default: return Register();
3271 }
3272}
3273
3274// Top-level FastEmit function.
3275
3276Register fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, Register Op0) override {
3277 switch (Opcode) {
3278 case ARMISD::CALL: return fastEmit_ARMISD_CALL_r(VT, RetVT, Op0);
3279 case ARMISD::CALL_NOLINK: return fastEmit_ARMISD_CALL_NOLINK_r(VT, RetVT, Op0);
3280 case ARMISD::CALL_PRED: return fastEmit_ARMISD_CALL_PRED_r(VT, RetVT, Op0);
3281 case ARMISD::CMPFPEw0: return fastEmit_ARMISD_CMPFPEw0_r(VT, RetVT, Op0);
3282 case ARMISD::CMPFPw0: return fastEmit_ARMISD_CMPFPw0_r(VT, RetVT, Op0);
3283 case ARMISD::VADDVs: return fastEmit_ARMISD_VADDVs_r(VT, RetVT, Op0);
3284 case ARMISD::VADDVu: return fastEmit_ARMISD_VADDVu_r(VT, RetVT, Op0);
3285 case ARMISD::VDUP: return fastEmit_ARMISD_VDUP_r(VT, RetVT, Op0);
3286 case ARMISD::VMOVSR: return fastEmit_ARMISD_VMOVSR_r(VT, RetVT, Op0);
3287 case ARMISD::VMOVhr: return fastEmit_ARMISD_VMOVhr_r(VT, RetVT, Op0);
3288 case ARMISD::VMOVrh: return fastEmit_ARMISD_VMOVrh_r(VT, RetVT, Op0);
3289 case ARMISD::VREV16: return fastEmit_ARMISD_VREV16_r(VT, RetVT, Op0);
3290 case ARMISD::VREV32: return fastEmit_ARMISD_VREV32_r(VT, RetVT, Op0);
3291 case ARMISD::VREV64: return fastEmit_ARMISD_VREV64_r(VT, RetVT, Op0);
3292 case ARMISD::WIN__DBZCHK: return fastEmit_ARMISD_WIN__DBZCHK_r(VT, RetVT, Op0);
3293 case ARMISD::tSECALL: return fastEmit_ARMISD_tSECALL_r(VT, RetVT, Op0);
3294 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
3295 case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0);
3296 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
3297 case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0);
3298 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0);
3299 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
3300 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
3301 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
3302 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
3303 case ISD::FCEIL: return fastEmit_ISD_FCEIL_r(VT, RetVT, Op0);
3304 case ISD::FFLOOR: return fastEmit_ISD_FFLOOR_r(VT, RetVT, Op0);
3305 case ISD::FNEARBYINT: return fastEmit_ISD_FNEARBYINT_r(VT, RetVT, Op0);
3306 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
3307 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
3308 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
3309 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
3310 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
3311 case ISD::FRINT: return fastEmit_ISD_FRINT_r(VT, RetVT, Op0);
3312 case ISD::FROUND: return fastEmit_ISD_FROUND_r(VT, RetVT, Op0);
3313 case ISD::FROUNDEVEN: return fastEmit_ISD_FROUNDEVEN_r(VT, RetVT, Op0);
3314 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
3315 case ISD::FTRUNC: return fastEmit_ISD_FTRUNC_r(VT, RetVT, Op0);
3316 case ISD::SET_FPENV: return fastEmit_ISD_SET_FPENV_r(VT, RetVT, Op0);
3317 case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0);
3318 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
3319 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
3320 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
3321 case ISD::VECREDUCE_ADD: return fastEmit_ISD_VECREDUCE_ADD_r(VT, RetVT, Op0);
3322 case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0);
3323 default: return Register();
3324 }
3325}
3326
3327// FastEmit functions for ARMISD::CMP.
3328
3329Register fastEmit_ARMISD_CMP_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3330 if (RetVT.SimpleTy != MVT::i32)
3331 return Register();
3332 if ((Subtarget->isThumb2())) {
3333 return fastEmitInst_rr(MachineInstOpcode: ARM::t2CMPrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3334 }
3335 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
3336 return fastEmitInst_rr(MachineInstOpcode: ARM::tCMPr, RC: &ARM::tGPRRegClass, Op0, Op1);
3337 }
3338 if ((!Subtarget->isThumb())) {
3339 return fastEmitInst_rr(MachineInstOpcode: ARM::CMPrr, RC: &ARM::GPRRegClass, Op0, Op1);
3340 }
3341 return Register();
3342}
3343
3344Register fastEmit_ARMISD_CMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3345 switch (VT.SimpleTy) {
3346 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_rr(RetVT, Op0, Op1);
3347 default: return Register();
3348 }
3349}
3350
3351// FastEmit functions for ARMISD::CMPFP.
3352
3353Register fastEmit_ARMISD_CMPFP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
3354 if (RetVT.SimpleTy != MVT::i32)
3355 return Register();
3356 if ((Subtarget->hasFullFP16())) {
3357 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPH, RC: &ARM::HPRRegClass, Op0, Op1);
3358 }
3359 return Register();
3360}
3361
3362Register fastEmit_ARMISD_CMPFP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
3363 if (RetVT.SimpleTy != MVT::i32)
3364 return Register();
3365 if ((Subtarget->hasVFP2Base())) {
3366 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPS, RC: &ARM::SPRRegClass, Op0, Op1);
3367 }
3368 return Register();
3369}
3370
3371Register fastEmit_ARMISD_CMPFP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
3372 if (RetVT.SimpleTy != MVT::i32)
3373 return Register();
3374 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3375 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPD, RC: &ARM::DPRRegClass, Op0, Op1);
3376 }
3377 return Register();
3378}
3379
3380Register fastEmit_ARMISD_CMPFP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3381 switch (VT.SimpleTy) {
3382 case MVT::f16: return fastEmit_ARMISD_CMPFP_MVT_f16_rr(RetVT, Op0, Op1);
3383 case MVT::f32: return fastEmit_ARMISD_CMPFP_MVT_f32_rr(RetVT, Op0, Op1);
3384 case MVT::f64: return fastEmit_ARMISD_CMPFP_MVT_f64_rr(RetVT, Op0, Op1);
3385 default: return Register();
3386 }
3387}
3388
3389// FastEmit functions for ARMISD::CMPFPE.
3390
3391Register fastEmit_ARMISD_CMPFPE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
3392 if (RetVT.SimpleTy != MVT::i32)
3393 return Register();
3394 if ((Subtarget->hasFullFP16())) {
3395 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPEH, RC: &ARM::HPRRegClass, Op0, Op1);
3396 }
3397 return Register();
3398}
3399
3400Register fastEmit_ARMISD_CMPFPE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
3401 if (RetVT.SimpleTy != MVT::i32)
3402 return Register();
3403 if ((Subtarget->hasVFP2Base())) {
3404 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPES, RC: &ARM::SPRRegClass, Op0, Op1);
3405 }
3406 return Register();
3407}
3408
3409Register fastEmit_ARMISD_CMPFPE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
3410 if (RetVT.SimpleTy != MVT::i32)
3411 return Register();
3412 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3413 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPED, RC: &ARM::DPRRegClass, Op0, Op1);
3414 }
3415 return Register();
3416}
3417
3418Register fastEmit_ARMISD_CMPFPE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3419 switch (VT.SimpleTy) {
3420 case MVT::f16: return fastEmit_ARMISD_CMPFPE_MVT_f16_rr(RetVT, Op0, Op1);
3421 case MVT::f32: return fastEmit_ARMISD_CMPFPE_MVT_f32_rr(RetVT, Op0, Op1);
3422 case MVT::f64: return fastEmit_ARMISD_CMPFPE_MVT_f64_rr(RetVT, Op0, Op1);
3423 default: return Register();
3424 }
3425}
3426
3427// FastEmit functions for ARMISD::CMPZ.
3428
3429Register fastEmit_ARMISD_CMPZ_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3430 if (RetVT.SimpleTy != MVT::i32)
3431 return Register();
3432 if ((Subtarget->isThumb2())) {
3433 return fastEmitInst_rr(MachineInstOpcode: ARM::t2CMPrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3434 }
3435 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
3436 return fastEmitInst_rr(MachineInstOpcode: ARM::tCMPr, RC: &ARM::tGPRRegClass, Op0, Op1);
3437 }
3438 if ((!Subtarget->isThumb())) {
3439 return fastEmitInst_rr(MachineInstOpcode: ARM::CMPrr, RC: &ARM::GPRRegClass, Op0, Op1);
3440 }
3441 return Register();
3442}
3443
3444Register fastEmit_ARMISD_CMPZ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3445 switch (VT.SimpleTy) {
3446 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_rr(RetVT, Op0, Op1);
3447 default: return Register();
3448 }
3449}
3450
3451// FastEmit functions for ARMISD::EH_SJLJ_LONGJMP.
3452
3453Register fastEmit_ARMISD_EH_SJLJ_LONGJMP_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3454 if (RetVT.SimpleTy != MVT::isVoid)
3455 return Register();
3456 if ((Subtarget->isThumb()) && (Subtarget->isTargetWindows())) {
3457 return fastEmitInst_rr(MachineInstOpcode: ARM::tInt_WIN_eh_sjlj_longjmp, RC: &ARM::GPRRegClass, Op0, Op1);
3458 }
3459 if ((!Subtarget->isTargetWindows()) && (Subtarget->isThumb())) {
3460 return fastEmitInst_rr(MachineInstOpcode: ARM::tInt_eh_sjlj_longjmp, RC: &ARM::tGPRRegClass, Op0, Op1);
3461 }
3462 if ((!Subtarget->isThumb())) {
3463 return fastEmitInst_rr(MachineInstOpcode: ARM::Int_eh_sjlj_longjmp, RC: &ARM::GPRRegClass, Op0, Op1);
3464 }
3465 return Register();
3466}
3467
3468Register fastEmit_ARMISD_EH_SJLJ_LONGJMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3469 switch (VT.SimpleTy) {
3470 case MVT::i32: return fastEmit_ARMISD_EH_SJLJ_LONGJMP_MVT_i32_rr(RetVT, Op0, Op1);
3471 default: return Register();
3472 }
3473}
3474
3475// FastEmit functions for ARMISD::EH_SJLJ_SETJMP.
3476
3477Register fastEmit_ARMISD_EH_SJLJ_SETJMP_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3478 if (RetVT.SimpleTy != MVT::i32)
3479 return Register();
3480 if ((Subtarget->isThumb2()) && (!Subtarget->hasVFP2Base())) {
3481 return fastEmitInst_rr(MachineInstOpcode: ARM::t2Int_eh_sjlj_setjmp_nofp, RC: &ARM::tGPRRegClass, Op0, Op1);
3482 }
3483 if ((Subtarget->hasVFP2Base()) && (Subtarget->isThumb2())) {
3484 return fastEmitInst_rr(MachineInstOpcode: ARM::t2Int_eh_sjlj_setjmp, RC: &ARM::tGPRRegClass, Op0, Op1);
3485 }
3486 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
3487 return fastEmitInst_rr(MachineInstOpcode: ARM::tInt_eh_sjlj_setjmp, RC: &ARM::tGPRRegClass, Op0, Op1);
3488 }
3489 if ((!Subtarget->isThumb()) && (!Subtarget->hasVFP2Base())) {
3490 return fastEmitInst_rr(MachineInstOpcode: ARM::Int_eh_sjlj_setjmp_nofp, RC: &ARM::GPRRegClass, Op0, Op1);
3491 }
3492 if ((Subtarget->hasVFP2Base()) && (!Subtarget->isThumb())) {
3493 return fastEmitInst_rr(MachineInstOpcode: ARM::Int_eh_sjlj_setjmp, RC: &ARM::GPRRegClass, Op0, Op1);
3494 }
3495 return Register();
3496}
3497
3498Register fastEmit_ARMISD_EH_SJLJ_SETJMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3499 switch (VT.SimpleTy) {
3500 case MVT::i32: return fastEmit_ARMISD_EH_SJLJ_SETJMP_MVT_i32_rr(RetVT, Op0, Op1);
3501 default: return Register();
3502 }
3503}
3504
3505// FastEmit functions for ARMISD::QADD16b.
3506
3507Register fastEmit_ARMISD_QADD16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3508 if (RetVT.SimpleTy != MVT::i32)
3509 return Register();
3510 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3511 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QADD16, RC: &ARM::rGPRRegClass, Op0, Op1);
3512 }
3513 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3514 return fastEmitInst_rr(MachineInstOpcode: ARM::QADD16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3515 }
3516 return Register();
3517}
3518
3519Register fastEmit_ARMISD_QADD16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3520 switch (VT.SimpleTy) {
3521 case MVT::i32: return fastEmit_ARMISD_QADD16b_MVT_i32_rr(RetVT, Op0, Op1);
3522 default: return Register();
3523 }
3524}
3525
3526// FastEmit functions for ARMISD::QADD8b.
3527
3528Register fastEmit_ARMISD_QADD8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3529 if (RetVT.SimpleTy != MVT::i32)
3530 return Register();
3531 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3532 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QADD8, RC: &ARM::rGPRRegClass, Op0, Op1);
3533 }
3534 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3535 return fastEmitInst_rr(MachineInstOpcode: ARM::QADD8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3536 }
3537 return Register();
3538}
3539
3540Register fastEmit_ARMISD_QADD8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3541 switch (VT.SimpleTy) {
3542 case MVT::i32: return fastEmit_ARMISD_QADD8b_MVT_i32_rr(RetVT, Op0, Op1);
3543 default: return Register();
3544 }
3545}
3546
3547// FastEmit functions for ARMISD::QSUB16b.
3548
3549Register fastEmit_ARMISD_QSUB16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3550 if (RetVT.SimpleTy != MVT::i32)
3551 return Register();
3552 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3553 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QSUB16, RC: &ARM::rGPRRegClass, Op0, Op1);
3554 }
3555 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3556 return fastEmitInst_rr(MachineInstOpcode: ARM::QSUB16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3557 }
3558 return Register();
3559}
3560
3561Register fastEmit_ARMISD_QSUB16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3562 switch (VT.SimpleTy) {
3563 case MVT::i32: return fastEmit_ARMISD_QSUB16b_MVT_i32_rr(RetVT, Op0, Op1);
3564 default: return Register();
3565 }
3566}
3567
3568// FastEmit functions for ARMISD::QSUB8b.
3569
3570Register fastEmit_ARMISD_QSUB8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3571 if (RetVT.SimpleTy != MVT::i32)
3572 return Register();
3573 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3574 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QSUB8, RC: &ARM::rGPRRegClass, Op0, Op1);
3575 }
3576 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3577 return fastEmitInst_rr(MachineInstOpcode: ARM::QSUB8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3578 }
3579 return Register();
3580}
3581
3582Register fastEmit_ARMISD_QSUB8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3583 switch (VT.SimpleTy) {
3584 case MVT::i32: return fastEmit_ARMISD_QSUB8b_MVT_i32_rr(RetVT, Op0, Op1);
3585 default: return Register();
3586 }
3587}
3588
3589// FastEmit functions for ARMISD::SMULWB.
3590
3591Register fastEmit_ARMISD_SMULWB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3592 if (RetVT.SimpleTy != MVT::i32)
3593 return Register();
3594 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3595 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SMULWB, RC: &ARM::rGPRRegClass, Op0, Op1);
3596 }
3597 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
3598 return fastEmitInst_rr(MachineInstOpcode: ARM::SMULWB, RC: &ARM::GPRRegClass, Op0, Op1);
3599 }
3600 return Register();
3601}
3602
3603Register fastEmit_ARMISD_SMULWB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3604 switch (VT.SimpleTy) {
3605 case MVT::i32: return fastEmit_ARMISD_SMULWB_MVT_i32_rr(RetVT, Op0, Op1);
3606 default: return Register();
3607 }
3608}
3609
3610// FastEmit functions for ARMISD::SMULWT.
3611
3612Register fastEmit_ARMISD_SMULWT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3613 if (RetVT.SimpleTy != MVT::i32)
3614 return Register();
3615 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3616 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SMULWT, RC: &ARM::rGPRRegClass, Op0, Op1);
3617 }
3618 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
3619 return fastEmitInst_rr(MachineInstOpcode: ARM::SMULWT, RC: &ARM::GPRRegClass, Op0, Op1);
3620 }
3621 return Register();
3622}
3623
3624Register fastEmit_ARMISD_SMULWT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3625 switch (VT.SimpleTy) {
3626 case MVT::i32: return fastEmit_ARMISD_SMULWT_MVT_i32_rr(RetVT, Op0, Op1);
3627 default: return Register();
3628 }
3629}
3630
3631// FastEmit functions for ARMISD::UQADD16b.
3632
3633Register fastEmit_ARMISD_UQADD16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3634 if (RetVT.SimpleTy != MVT::i32)
3635 return Register();
3636 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3637 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQADD16, RC: &ARM::rGPRRegClass, Op0, Op1);
3638 }
3639 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3640 return fastEmitInst_rr(MachineInstOpcode: ARM::UQADD16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3641 }
3642 return Register();
3643}
3644
3645Register fastEmit_ARMISD_UQADD16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3646 switch (VT.SimpleTy) {
3647 case MVT::i32: return fastEmit_ARMISD_UQADD16b_MVT_i32_rr(RetVT, Op0, Op1);
3648 default: return Register();
3649 }
3650}
3651
3652// FastEmit functions for ARMISD::UQADD8b.
3653
3654Register fastEmit_ARMISD_UQADD8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3655 if (RetVT.SimpleTy != MVT::i32)
3656 return Register();
3657 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3658 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQADD8, RC: &ARM::rGPRRegClass, Op0, Op1);
3659 }
3660 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3661 return fastEmitInst_rr(MachineInstOpcode: ARM::UQADD8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3662 }
3663 return Register();
3664}
3665
3666Register fastEmit_ARMISD_UQADD8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3667 switch (VT.SimpleTy) {
3668 case MVT::i32: return fastEmit_ARMISD_UQADD8b_MVT_i32_rr(RetVT, Op0, Op1);
3669 default: return Register();
3670 }
3671}
3672
3673// FastEmit functions for ARMISD::UQSUB16b.
3674
3675Register fastEmit_ARMISD_UQSUB16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3676 if (RetVT.SimpleTy != MVT::i32)
3677 return Register();
3678 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3679 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQSUB16, RC: &ARM::rGPRRegClass, Op0, Op1);
3680 }
3681 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3682 return fastEmitInst_rr(MachineInstOpcode: ARM::UQSUB16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3683 }
3684 return Register();
3685}
3686
3687Register fastEmit_ARMISD_UQSUB16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3688 switch (VT.SimpleTy) {
3689 case MVT::i32: return fastEmit_ARMISD_UQSUB16b_MVT_i32_rr(RetVT, Op0, Op1);
3690 default: return Register();
3691 }
3692}
3693
3694// FastEmit functions for ARMISD::UQSUB8b.
3695
3696Register fastEmit_ARMISD_UQSUB8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3697 if (RetVT.SimpleTy != MVT::i32)
3698 return Register();
3699 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3700 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQSUB8, RC: &ARM::rGPRRegClass, Op0, Op1);
3701 }
3702 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3703 return fastEmitInst_rr(MachineInstOpcode: ARM::UQSUB8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3704 }
3705 return Register();
3706}
3707
3708Register fastEmit_ARMISD_UQSUB8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3709 switch (VT.SimpleTy) {
3710 case MVT::i32: return fastEmit_ARMISD_UQSUB8b_MVT_i32_rr(RetVT, Op0, Op1);
3711 default: return Register();
3712 }
3713}
3714
3715// FastEmit functions for ARMISD::VMLAVs.
3716
3717Register fastEmit_ARMISD_VMLAVs_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
3718 if (RetVT.SimpleTy != MVT::i32)
3719 return Register();
3720 if ((Subtarget->hasMVEIntegerOps())) {
3721 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVs8, RC: &ARM::tGPREvenRegClass, Op0, Op1);
3722 }
3723 return Register();
3724}
3725
3726Register fastEmit_ARMISD_VMLAVs_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
3727 if (RetVT.SimpleTy != MVT::i32)
3728 return Register();
3729 if ((Subtarget->hasMVEIntegerOps())) {
3730 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVs16, RC: &ARM::tGPREvenRegClass, Op0, Op1);
3731 }
3732 return Register();
3733}
3734
3735Register fastEmit_ARMISD_VMLAVs_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3736 switch (VT.SimpleTy) {
3737 case MVT::v16i8: return fastEmit_ARMISD_VMLAVs_MVT_v16i8_rr(RetVT, Op0, Op1);
3738 case MVT::v8i16: return fastEmit_ARMISD_VMLAVs_MVT_v8i16_rr(RetVT, Op0, Op1);
3739 default: return Register();
3740 }
3741}
3742
3743// FastEmit functions for ARMISD::VMLAVu.
3744
3745Register fastEmit_ARMISD_VMLAVu_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
3746 if (RetVT.SimpleTy != MVT::i32)
3747 return Register();
3748 if ((Subtarget->hasMVEIntegerOps())) {
3749 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVu8, RC: &ARM::tGPREvenRegClass, Op0, Op1);
3750 }
3751 return Register();
3752}
3753
3754Register fastEmit_ARMISD_VMLAVu_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
3755 if (RetVT.SimpleTy != MVT::i32)
3756 return Register();
3757 if ((Subtarget->hasMVEIntegerOps())) {
3758 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVu16, RC: &ARM::tGPREvenRegClass, Op0, Op1);
3759 }
3760 return Register();
3761}
3762
3763Register fastEmit_ARMISD_VMLAVu_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3764 switch (VT.SimpleTy) {
3765 case MVT::v16i8: return fastEmit_ARMISD_VMLAVu_MVT_v16i8_rr(RetVT, Op0, Op1);
3766 case MVT::v8i16: return fastEmit_ARMISD_VMLAVu_MVT_v8i16_rr(RetVT, Op0, Op1);
3767 default: return Register();
3768 }
3769}
3770
3771// FastEmit functions for ARMISD::VMOVDRR.
3772
3773Register fastEmit_ARMISD_VMOVDRR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
3774 if (RetVT.SimpleTy != MVT::f64)
3775 return Register();
3776 if ((Subtarget->hasFPRegs())) {
3777 return fastEmitInst_rr(MachineInstOpcode: ARM::VMOVDRR, RC: &ARM::DPRRegClass, Op0, Op1);
3778 }
3779 return Register();
3780}
3781
3782Register fastEmit_ARMISD_VMOVDRR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3783 switch (VT.SimpleTy) {
3784 case MVT::i32: return fastEmit_ARMISD_VMOVDRR_MVT_i32_rr(RetVT, Op0, Op1);
3785 default: return Register();
3786 }
3787}
3788
3789// FastEmit functions for ARMISD::VMULLs.
3790
3791Register fastEmit_ARMISD_VMULLs_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
3792 if (RetVT.SimpleTy != MVT::v8i16)
3793 return Register();
3794 if ((Subtarget->hasNEON())) {
3795 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
3796 }
3797 return Register();
3798}
3799
3800Register fastEmit_ARMISD_VMULLs_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
3801 if (RetVT.SimpleTy != MVT::v4i32)
3802 return Register();
3803 if ((Subtarget->hasNEON())) {
3804 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
3805 }
3806 return Register();
3807}
3808
3809Register fastEmit_ARMISD_VMULLs_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
3810 if (RetVT.SimpleTy != MVT::v2i64)
3811 return Register();
3812 if ((Subtarget->hasNEON())) {
3813 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
3814 }
3815 return Register();
3816}
3817
3818Register fastEmit_ARMISD_VMULLs_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
3819 if (RetVT.SimpleTy != MVT::v2i64)
3820 return Register();
3821 if ((Subtarget->hasMVEIntegerOps())) {
3822 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULLBs32, RC: &ARM::MQPRRegClass, Op0, Op1);
3823 }
3824 return Register();
3825}
3826
3827Register fastEmit_ARMISD_VMULLs_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3828 switch (VT.SimpleTy) {
3829 case MVT::v8i8: return fastEmit_ARMISD_VMULLs_MVT_v8i8_rr(RetVT, Op0, Op1);
3830 case MVT::v4i16: return fastEmit_ARMISD_VMULLs_MVT_v4i16_rr(RetVT, Op0, Op1);
3831 case MVT::v2i32: return fastEmit_ARMISD_VMULLs_MVT_v2i32_rr(RetVT, Op0, Op1);
3832 case MVT::v4i32: return fastEmit_ARMISD_VMULLs_MVT_v4i32_rr(RetVT, Op0, Op1);
3833 default: return Register();
3834 }
3835}
3836
3837// FastEmit functions for ARMISD::VMULLu.
3838
3839Register fastEmit_ARMISD_VMULLu_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
3840 if (RetVT.SimpleTy != MVT::v8i16)
3841 return Register();
3842 if ((Subtarget->hasNEON())) {
3843 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
3844 }
3845 return Register();
3846}
3847
3848Register fastEmit_ARMISD_VMULLu_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
3849 if (RetVT.SimpleTy != MVT::v4i32)
3850 return Register();
3851 if ((Subtarget->hasNEON())) {
3852 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
3853 }
3854 return Register();
3855}
3856
3857Register fastEmit_ARMISD_VMULLu_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
3858 if (RetVT.SimpleTy != MVT::v2i64)
3859 return Register();
3860 if ((Subtarget->hasNEON())) {
3861 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
3862 }
3863 return Register();
3864}
3865
3866Register fastEmit_ARMISD_VMULLu_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
3867 if (RetVT.SimpleTy != MVT::v2i64)
3868 return Register();
3869 if ((Subtarget->hasMVEIntegerOps())) {
3870 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULLBu32, RC: &ARM::MQPRRegClass, Op0, Op1);
3871 }
3872 return Register();
3873}
3874
3875Register fastEmit_ARMISD_VMULLu_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3876 switch (VT.SimpleTy) {
3877 case MVT::v8i8: return fastEmit_ARMISD_VMULLu_MVT_v8i8_rr(RetVT, Op0, Op1);
3878 case MVT::v4i16: return fastEmit_ARMISD_VMULLu_MVT_v4i16_rr(RetVT, Op0, Op1);
3879 case MVT::v2i32: return fastEmit_ARMISD_VMULLu_MVT_v2i32_rr(RetVT, Op0, Op1);
3880 case MVT::v4i32: return fastEmit_ARMISD_VMULLu_MVT_v4i32_rr(RetVT, Op0, Op1);
3881 default: return Register();
3882 }
3883}
3884
3885// FastEmit functions for ARMISD::VQDMULH.
3886
3887Register fastEmit_ARMISD_VQDMULH_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
3888 if (RetVT.SimpleTy != MVT::v16i8)
3889 return Register();
3890 if ((Subtarget->hasMVEIntegerOps())) {
3891 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQDMULHi8, RC: &ARM::MQPRRegClass, Op0, Op1);
3892 }
3893 return Register();
3894}
3895
3896Register fastEmit_ARMISD_VQDMULH_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
3897 if (RetVT.SimpleTy != MVT::v8i16)
3898 return Register();
3899 if ((Subtarget->hasMVEIntegerOps())) {
3900 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQDMULHi16, RC: &ARM::MQPRRegClass, Op0, Op1);
3901 }
3902 return Register();
3903}
3904
3905Register fastEmit_ARMISD_VQDMULH_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
3906 if (RetVT.SimpleTy != MVT::v4i32)
3907 return Register();
3908 if ((Subtarget->hasMVEIntegerOps())) {
3909 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQDMULHi32, RC: &ARM::MQPRRegClass, Op0, Op1);
3910 }
3911 return Register();
3912}
3913
3914Register fastEmit_ARMISD_VQDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
3915 switch (VT.SimpleTy) {
3916 case MVT::v16i8: return fastEmit_ARMISD_VQDMULH_MVT_v16i8_rr(RetVT, Op0, Op1);
3917 case MVT::v8i16: return fastEmit_ARMISD_VQDMULH_MVT_v8i16_rr(RetVT, Op0, Op1);
3918 case MVT::v4i32: return fastEmit_ARMISD_VQDMULH_MVT_v4i32_rr(RetVT, Op0, Op1);
3919 default: return Register();
3920 }
3921}
3922
3923// FastEmit functions for ARMISD::VSHLs.
3924
3925Register fastEmit_ARMISD_VSHLs_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
3926 if (RetVT.SimpleTy != MVT::v8i8)
3927 return Register();
3928 if ((Subtarget->hasNEON())) {
3929 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
3930 }
3931 return Register();
3932}
3933
3934Register fastEmit_ARMISD_VSHLs_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
3935 if (RetVT.SimpleTy != MVT::v16i8)
3936 return Register();
3937 if ((Subtarget->hasMVEIntegerOps())) {
3938 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecs8, RC: &ARM::MQPRRegClass, Op0, Op1);
3939 }
3940 if ((Subtarget->hasNEON())) {
3941 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
3942 }
3943 return Register();
3944}
3945
3946Register fastEmit_ARMISD_VSHLs_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
3947 if (RetVT.SimpleTy != MVT::v4i16)
3948 return Register();
3949 if ((Subtarget->hasNEON())) {
3950 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
3951 }
3952 return Register();
3953}
3954
3955Register fastEmit_ARMISD_VSHLs_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
3956 if (RetVT.SimpleTy != MVT::v8i16)
3957 return Register();
3958 if ((Subtarget->hasMVEIntegerOps())) {
3959 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecs16, RC: &ARM::MQPRRegClass, Op0, Op1);
3960 }
3961 if ((Subtarget->hasNEON())) {
3962 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
3963 }
3964 return Register();
3965}
3966
3967Register fastEmit_ARMISD_VSHLs_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
3968 if (RetVT.SimpleTy != MVT::v2i32)
3969 return Register();
3970 if ((Subtarget->hasNEON())) {
3971 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
3972 }
3973 return Register();
3974}
3975
3976Register fastEmit_ARMISD_VSHLs_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
3977 if (RetVT.SimpleTy != MVT::v4i32)
3978 return Register();
3979 if ((Subtarget->hasMVEIntegerOps())) {
3980 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecs32, RC: &ARM::MQPRRegClass, Op0, Op1);
3981 }
3982 if ((Subtarget->hasNEON())) {
3983 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
3984 }
3985 return Register();
3986}
3987
3988Register fastEmit_ARMISD_VSHLs_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
3989 if (RetVT.SimpleTy != MVT::v1i64)
3990 return Register();
3991 if ((Subtarget->hasNEON())) {
3992 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
3993 }
3994 return Register();
3995}
3996
3997Register fastEmit_ARMISD_VSHLs_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
3998 if (RetVT.SimpleTy != MVT::v2i64)
3999 return Register();
4000 if ((Subtarget->hasNEON())) {
4001 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4002 }
4003 return Register();
4004}
4005
4006Register fastEmit_ARMISD_VSHLs_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4007 switch (VT.SimpleTy) {
4008 case MVT::v8i8: return fastEmit_ARMISD_VSHLs_MVT_v8i8_rr(RetVT, Op0, Op1);
4009 case MVT::v16i8: return fastEmit_ARMISD_VSHLs_MVT_v16i8_rr(RetVT, Op0, Op1);
4010 case MVT::v4i16: return fastEmit_ARMISD_VSHLs_MVT_v4i16_rr(RetVT, Op0, Op1);
4011 case MVT::v8i16: return fastEmit_ARMISD_VSHLs_MVT_v8i16_rr(RetVT, Op0, Op1);
4012 case MVT::v2i32: return fastEmit_ARMISD_VSHLs_MVT_v2i32_rr(RetVT, Op0, Op1);
4013 case MVT::v4i32: return fastEmit_ARMISD_VSHLs_MVT_v4i32_rr(RetVT, Op0, Op1);
4014 case MVT::v1i64: return fastEmit_ARMISD_VSHLs_MVT_v1i64_rr(RetVT, Op0, Op1);
4015 case MVT::v2i64: return fastEmit_ARMISD_VSHLs_MVT_v2i64_rr(RetVT, Op0, Op1);
4016 default: return Register();
4017 }
4018}
4019
4020// FastEmit functions for ARMISD::VSHLu.
4021
4022Register fastEmit_ARMISD_VSHLu_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4023 if (RetVT.SimpleTy != MVT::v8i8)
4024 return Register();
4025 if ((Subtarget->hasNEON())) {
4026 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4027 }
4028 return Register();
4029}
4030
4031Register fastEmit_ARMISD_VSHLu_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4032 if (RetVT.SimpleTy != MVT::v16i8)
4033 return Register();
4034 if ((Subtarget->hasMVEIntegerOps())) {
4035 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecu8, RC: &ARM::MQPRRegClass, Op0, Op1);
4036 }
4037 if ((Subtarget->hasNEON())) {
4038 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4039 }
4040 return Register();
4041}
4042
4043Register fastEmit_ARMISD_VSHLu_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4044 if (RetVT.SimpleTy != MVT::v4i16)
4045 return Register();
4046 if ((Subtarget->hasNEON())) {
4047 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4048 }
4049 return Register();
4050}
4051
4052Register fastEmit_ARMISD_VSHLu_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4053 if (RetVT.SimpleTy != MVT::v8i16)
4054 return Register();
4055 if ((Subtarget->hasMVEIntegerOps())) {
4056 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecu16, RC: &ARM::MQPRRegClass, Op0, Op1);
4057 }
4058 if ((Subtarget->hasNEON())) {
4059 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4060 }
4061 return Register();
4062}
4063
4064Register fastEmit_ARMISD_VSHLu_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4065 if (RetVT.SimpleTy != MVT::v2i32)
4066 return Register();
4067 if ((Subtarget->hasNEON())) {
4068 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4069 }
4070 return Register();
4071}
4072
4073Register fastEmit_ARMISD_VSHLu_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4074 if (RetVT.SimpleTy != MVT::v4i32)
4075 return Register();
4076 if ((Subtarget->hasMVEIntegerOps())) {
4077 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecu32, RC: &ARM::MQPRRegClass, Op0, Op1);
4078 }
4079 if ((Subtarget->hasNEON())) {
4080 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4081 }
4082 return Register();
4083}
4084
4085Register fastEmit_ARMISD_VSHLu_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
4086 if (RetVT.SimpleTy != MVT::v1i64)
4087 return Register();
4088 if ((Subtarget->hasNEON())) {
4089 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
4090 }
4091 return Register();
4092}
4093
4094Register fastEmit_ARMISD_VSHLu_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
4095 if (RetVT.SimpleTy != MVT::v2i64)
4096 return Register();
4097 if ((Subtarget->hasNEON())) {
4098 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4099 }
4100 return Register();
4101}
4102
4103Register fastEmit_ARMISD_VSHLu_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4104 switch (VT.SimpleTy) {
4105 case MVT::v8i8: return fastEmit_ARMISD_VSHLu_MVT_v8i8_rr(RetVT, Op0, Op1);
4106 case MVT::v16i8: return fastEmit_ARMISD_VSHLu_MVT_v16i8_rr(RetVT, Op0, Op1);
4107 case MVT::v4i16: return fastEmit_ARMISD_VSHLu_MVT_v4i16_rr(RetVT, Op0, Op1);
4108 case MVT::v8i16: return fastEmit_ARMISD_VSHLu_MVT_v8i16_rr(RetVT, Op0, Op1);
4109 case MVT::v2i32: return fastEmit_ARMISD_VSHLu_MVT_v2i32_rr(RetVT, Op0, Op1);
4110 case MVT::v4i32: return fastEmit_ARMISD_VSHLu_MVT_v4i32_rr(RetVT, Op0, Op1);
4111 case MVT::v1i64: return fastEmit_ARMISD_VSHLu_MVT_v1i64_rr(RetVT, Op0, Op1);
4112 case MVT::v2i64: return fastEmit_ARMISD_VSHLu_MVT_v2i64_rr(RetVT, Op0, Op1);
4113 default: return Register();
4114 }
4115}
4116
4117// FastEmit functions for ARMISD::VTBL1.
4118
4119Register fastEmit_ARMISD_VTBL1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4120 if (RetVT.SimpleTy != MVT::v8i8)
4121 return Register();
4122 if ((Subtarget->hasNEON())) {
4123 return fastEmitInst_rr(MachineInstOpcode: ARM::VTBL1, RC: &ARM::DPRRegClass, Op0, Op1);
4124 }
4125 return Register();
4126}
4127
4128Register fastEmit_ARMISD_VTBL1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4129 switch (VT.SimpleTy) {
4130 case MVT::v8i8: return fastEmit_ARMISD_VTBL1_MVT_v8i8_rr(RetVT, Op0, Op1);
4131 default: return Register();
4132 }
4133}
4134
4135// FastEmit functions for ARMISD::VTST.
4136
4137Register fastEmit_ARMISD_VTST_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4138 if (RetVT.SimpleTy != MVT::v8i8)
4139 return Register();
4140 if ((Subtarget->hasNEON())) {
4141 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4142 }
4143 return Register();
4144}
4145
4146Register fastEmit_ARMISD_VTST_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4147 if (RetVT.SimpleTy != MVT::v16i8)
4148 return Register();
4149 if ((Subtarget->hasNEON())) {
4150 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4151 }
4152 return Register();
4153}
4154
4155Register fastEmit_ARMISD_VTST_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4156 if (RetVT.SimpleTy != MVT::v4i16)
4157 return Register();
4158 if ((Subtarget->hasNEON())) {
4159 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4160 }
4161 return Register();
4162}
4163
4164Register fastEmit_ARMISD_VTST_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4165 if (RetVT.SimpleTy != MVT::v8i16)
4166 return Register();
4167 if ((Subtarget->hasNEON())) {
4168 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4169 }
4170 return Register();
4171}
4172
4173Register fastEmit_ARMISD_VTST_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4174 if (RetVT.SimpleTy != MVT::v2i32)
4175 return Register();
4176 if ((Subtarget->hasNEON())) {
4177 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4178 }
4179 return Register();
4180}
4181
4182Register fastEmit_ARMISD_VTST_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4183 if (RetVT.SimpleTy != MVT::v4i32)
4184 return Register();
4185 if ((Subtarget->hasNEON())) {
4186 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4187 }
4188 return Register();
4189}
4190
4191Register fastEmit_ARMISD_VTST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4192 switch (VT.SimpleTy) {
4193 case MVT::v8i8: return fastEmit_ARMISD_VTST_MVT_v8i8_rr(RetVT, Op0, Op1);
4194 case MVT::v16i8: return fastEmit_ARMISD_VTST_MVT_v16i8_rr(RetVT, Op0, Op1);
4195 case MVT::v4i16: return fastEmit_ARMISD_VTST_MVT_v4i16_rr(RetVT, Op0, Op1);
4196 case MVT::v8i16: return fastEmit_ARMISD_VTST_MVT_v8i16_rr(RetVT, Op0, Op1);
4197 case MVT::v2i32: return fastEmit_ARMISD_VTST_MVT_v2i32_rr(RetVT, Op0, Op1);
4198 case MVT::v4i32: return fastEmit_ARMISD_VTST_MVT_v4i32_rr(RetVT, Op0, Op1);
4199 default: return Register();
4200 }
4201}
4202
4203// FastEmit functions for ISD::ABDS.
4204
4205Register fastEmit_ISD_ABDS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4206 if (RetVT.SimpleTy != MVT::v8i8)
4207 return Register();
4208 if ((Subtarget->hasNEON())) {
4209 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4210 }
4211 return Register();
4212}
4213
4214Register fastEmit_ISD_ABDS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4215 if (RetVT.SimpleTy != MVT::v16i8)
4216 return Register();
4217 if ((Subtarget->hasMVEIntegerOps())) {
4218 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
4219 }
4220 if ((Subtarget->hasNEON())) {
4221 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4222 }
4223 return Register();
4224}
4225
4226Register fastEmit_ISD_ABDS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4227 if (RetVT.SimpleTy != MVT::v4i16)
4228 return Register();
4229 if ((Subtarget->hasNEON())) {
4230 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4231 }
4232 return Register();
4233}
4234
4235Register fastEmit_ISD_ABDS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4236 if (RetVT.SimpleTy != MVT::v8i16)
4237 return Register();
4238 if ((Subtarget->hasMVEIntegerOps())) {
4239 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
4240 }
4241 if ((Subtarget->hasNEON())) {
4242 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4243 }
4244 return Register();
4245}
4246
4247Register fastEmit_ISD_ABDS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4248 if (RetVT.SimpleTy != MVT::v2i32)
4249 return Register();
4250 if ((Subtarget->hasNEON())) {
4251 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4252 }
4253 return Register();
4254}
4255
4256Register fastEmit_ISD_ABDS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4257 if (RetVT.SimpleTy != MVT::v4i32)
4258 return Register();
4259 if ((Subtarget->hasMVEIntegerOps())) {
4260 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
4261 }
4262 if ((Subtarget->hasNEON())) {
4263 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4264 }
4265 return Register();
4266}
4267
4268Register fastEmit_ISD_ABDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4269 switch (VT.SimpleTy) {
4270 case MVT::v8i8: return fastEmit_ISD_ABDS_MVT_v8i8_rr(RetVT, Op0, Op1);
4271 case MVT::v16i8: return fastEmit_ISD_ABDS_MVT_v16i8_rr(RetVT, Op0, Op1);
4272 case MVT::v4i16: return fastEmit_ISD_ABDS_MVT_v4i16_rr(RetVT, Op0, Op1);
4273 case MVT::v8i16: return fastEmit_ISD_ABDS_MVT_v8i16_rr(RetVT, Op0, Op1);
4274 case MVT::v2i32: return fastEmit_ISD_ABDS_MVT_v2i32_rr(RetVT, Op0, Op1);
4275 case MVT::v4i32: return fastEmit_ISD_ABDS_MVT_v4i32_rr(RetVT, Op0, Op1);
4276 default: return Register();
4277 }
4278}
4279
4280// FastEmit functions for ISD::ABDU.
4281
4282Register fastEmit_ISD_ABDU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4283 if (RetVT.SimpleTy != MVT::v8i8)
4284 return Register();
4285 if ((Subtarget->hasNEON())) {
4286 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4287 }
4288 return Register();
4289}
4290
4291Register fastEmit_ISD_ABDU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4292 if (RetVT.SimpleTy != MVT::v16i8)
4293 return Register();
4294 if ((Subtarget->hasMVEIntegerOps())) {
4295 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
4296 }
4297 if ((Subtarget->hasNEON())) {
4298 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4299 }
4300 return Register();
4301}
4302
4303Register fastEmit_ISD_ABDU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4304 if (RetVT.SimpleTy != MVT::v4i16)
4305 return Register();
4306 if ((Subtarget->hasNEON())) {
4307 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4308 }
4309 return Register();
4310}
4311
4312Register fastEmit_ISD_ABDU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4313 if (RetVT.SimpleTy != MVT::v8i16)
4314 return Register();
4315 if ((Subtarget->hasMVEIntegerOps())) {
4316 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
4317 }
4318 if ((Subtarget->hasNEON())) {
4319 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4320 }
4321 return Register();
4322}
4323
4324Register fastEmit_ISD_ABDU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4325 if (RetVT.SimpleTy != MVT::v2i32)
4326 return Register();
4327 if ((Subtarget->hasNEON())) {
4328 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4329 }
4330 return Register();
4331}
4332
4333Register fastEmit_ISD_ABDU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4334 if (RetVT.SimpleTy != MVT::v4i32)
4335 return Register();
4336 if ((Subtarget->hasMVEIntegerOps())) {
4337 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
4338 }
4339 if ((Subtarget->hasNEON())) {
4340 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4341 }
4342 return Register();
4343}
4344
4345Register fastEmit_ISD_ABDU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4346 switch (VT.SimpleTy) {
4347 case MVT::v8i8: return fastEmit_ISD_ABDU_MVT_v8i8_rr(RetVT, Op0, Op1);
4348 case MVT::v16i8: return fastEmit_ISD_ABDU_MVT_v16i8_rr(RetVT, Op0, Op1);
4349 case MVT::v4i16: return fastEmit_ISD_ABDU_MVT_v4i16_rr(RetVT, Op0, Op1);
4350 case MVT::v8i16: return fastEmit_ISD_ABDU_MVT_v8i16_rr(RetVT, Op0, Op1);
4351 case MVT::v2i32: return fastEmit_ISD_ABDU_MVT_v2i32_rr(RetVT, Op0, Op1);
4352 case MVT::v4i32: return fastEmit_ISD_ABDU_MVT_v4i32_rr(RetVT, Op0, Op1);
4353 default: return Register();
4354 }
4355}
4356
4357// FastEmit functions for ISD::ADD.
4358
4359Register fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4360 if (RetVT.SimpleTy != MVT::i32)
4361 return Register();
4362 if ((Subtarget->isThumb2())) {
4363 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ADDrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4364 }
4365 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
4366 return fastEmitInst_rr(MachineInstOpcode: ARM::tADDrr, RC: &ARM::tGPRRegClass, Op0, Op1);
4367 }
4368 if ((!Subtarget->isThumb())) {
4369 return fastEmitInst_rr(MachineInstOpcode: ARM::ADDrr, RC: &ARM::GPRRegClass, Op0, Op1);
4370 }
4371 return Register();
4372}
4373
4374Register fastEmit_ISD_ADD_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4375 if (RetVT.SimpleTy != MVT::v8i8)
4376 return Register();
4377 if ((Subtarget->hasNEON())) {
4378 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4379 }
4380 return Register();
4381}
4382
4383Register fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4384 if (RetVT.SimpleTy != MVT::v16i8)
4385 return Register();
4386 if ((Subtarget->hasMVEIntegerOps())) {
4387 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDi8, RC: &ARM::MQPRRegClass, Op0, Op1);
4388 }
4389 if ((Subtarget->hasNEON())) {
4390 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4391 }
4392 return Register();
4393}
4394
4395Register fastEmit_ISD_ADD_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4396 if (RetVT.SimpleTy != MVT::v4i16)
4397 return Register();
4398 if ((Subtarget->hasNEON())) {
4399 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4400 }
4401 return Register();
4402}
4403
4404Register fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4405 if (RetVT.SimpleTy != MVT::v8i16)
4406 return Register();
4407 if ((Subtarget->hasMVEIntegerOps())) {
4408 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDi16, RC: &ARM::MQPRRegClass, Op0, Op1);
4409 }
4410 if ((Subtarget->hasNEON())) {
4411 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4412 }
4413 return Register();
4414}
4415
4416Register fastEmit_ISD_ADD_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4417 if (RetVT.SimpleTy != MVT::v2i32)
4418 return Register();
4419 if ((Subtarget->hasNEON())) {
4420 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4421 }
4422 return Register();
4423}
4424
4425Register fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4426 if (RetVT.SimpleTy != MVT::v4i32)
4427 return Register();
4428 if ((Subtarget->hasMVEIntegerOps())) {
4429 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDi32, RC: &ARM::MQPRRegClass, Op0, Op1);
4430 }
4431 if ((Subtarget->hasNEON())) {
4432 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4433 }
4434 return Register();
4435}
4436
4437Register fastEmit_ISD_ADD_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
4438 if (RetVT.SimpleTy != MVT::v1i64)
4439 return Register();
4440 if ((Subtarget->hasNEON())) {
4441 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
4442 }
4443 return Register();
4444}
4445
4446Register fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
4447 if (RetVT.SimpleTy != MVT::v2i64)
4448 return Register();
4449 if ((Subtarget->hasNEON())) {
4450 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4451 }
4452 return Register();
4453}
4454
4455Register fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4456 switch (VT.SimpleTy) {
4457 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
4458 case MVT::v8i8: return fastEmit_ISD_ADD_MVT_v8i8_rr(RetVT, Op0, Op1);
4459 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
4460 case MVT::v4i16: return fastEmit_ISD_ADD_MVT_v4i16_rr(RetVT, Op0, Op1);
4461 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
4462 case MVT::v2i32: return fastEmit_ISD_ADD_MVT_v2i32_rr(RetVT, Op0, Op1);
4463 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
4464 case MVT::v1i64: return fastEmit_ISD_ADD_MVT_v1i64_rr(RetVT, Op0, Op1);
4465 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
4466 default: return Register();
4467 }
4468}
4469
4470// FastEmit functions for ISD::AND.
4471
4472Register fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4473 if (RetVT.SimpleTy != MVT::i32)
4474 return Register();
4475 if ((Subtarget->isThumb2())) {
4476 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ANDrr, RC: &ARM::rGPRRegClass, Op0, Op1);
4477 }
4478 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
4479 return fastEmitInst_rr(MachineInstOpcode: ARM::tAND, RC: &ARM::tGPRRegClass, Op0, Op1);
4480 }
4481 if ((!Subtarget->isThumb())) {
4482 return fastEmitInst_rr(MachineInstOpcode: ARM::ANDrr, RC: &ARM::GPRRegClass, Op0, Op1);
4483 }
4484 return Register();
4485}
4486
4487Register fastEmit_ISD_AND_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4488 if (RetVT.SimpleTy != MVT::v8i8)
4489 return Register();
4490 if ((Subtarget->hasNEON())) {
4491 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
4492 }
4493 return Register();
4494}
4495
4496Register fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4497 if (RetVT.SimpleTy != MVT::v16i8)
4498 return Register();
4499 if ((Subtarget->hasMVEIntegerOps())) {
4500 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
4501 }
4502 if ((Subtarget->hasNEON())) {
4503 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
4504 }
4505 return Register();
4506}
4507
4508Register fastEmit_ISD_AND_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4509 if (RetVT.SimpleTy != MVT::v4i16)
4510 return Register();
4511 if ((Subtarget->hasNEON())) {
4512 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
4513 }
4514 return Register();
4515}
4516
4517Register fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4518 if (RetVT.SimpleTy != MVT::v8i16)
4519 return Register();
4520 if ((Subtarget->hasMVEIntegerOps())) {
4521 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
4522 }
4523 if ((Subtarget->hasNEON())) {
4524 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
4525 }
4526 return Register();
4527}
4528
4529Register fastEmit_ISD_AND_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4530 if (RetVT.SimpleTy != MVT::v2i32)
4531 return Register();
4532 if ((Subtarget->hasNEON())) {
4533 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
4534 }
4535 return Register();
4536}
4537
4538Register fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4539 if (RetVT.SimpleTy != MVT::v4i32)
4540 return Register();
4541 if ((Subtarget->hasMVEIntegerOps())) {
4542 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
4543 }
4544 if ((Subtarget->hasNEON())) {
4545 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
4546 }
4547 return Register();
4548}
4549
4550Register fastEmit_ISD_AND_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
4551 if (RetVT.SimpleTy != MVT::v1i64)
4552 return Register();
4553 if ((Subtarget->hasNEON())) {
4554 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
4555 }
4556 return Register();
4557}
4558
4559Register fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
4560 if (RetVT.SimpleTy != MVT::v2i64)
4561 return Register();
4562 if ((Subtarget->hasMVEIntegerOps())) {
4563 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
4564 }
4565 if ((Subtarget->hasNEON())) {
4566 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
4567 }
4568 return Register();
4569}
4570
4571Register fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4572 switch (VT.SimpleTy) {
4573 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
4574 case MVT::v8i8: return fastEmit_ISD_AND_MVT_v8i8_rr(RetVT, Op0, Op1);
4575 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
4576 case MVT::v4i16: return fastEmit_ISD_AND_MVT_v4i16_rr(RetVT, Op0, Op1);
4577 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
4578 case MVT::v2i32: return fastEmit_ISD_AND_MVT_v2i32_rr(RetVT, Op0, Op1);
4579 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
4580 case MVT::v1i64: return fastEmit_ISD_AND_MVT_v1i64_rr(RetVT, Op0, Op1);
4581 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
4582 default: return Register();
4583 }
4584}
4585
4586// FastEmit functions for ISD::AVGCEILS.
4587
4588Register fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4589 if (RetVT.SimpleTy != MVT::v16i8)
4590 return Register();
4591 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
4592}
4593
4594Register fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4595 if (RetVT.SimpleTy != MVT::v8i16)
4596 return Register();
4597 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
4598}
4599
4600Register fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4601 if (RetVT.SimpleTy != MVT::v4i32)
4602 return Register();
4603 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
4604}
4605
4606Register fastEmit_ISD_AVGCEILS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4607 switch (VT.SimpleTy) {
4608 case MVT::v16i8: return fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(RetVT, Op0, Op1);
4609 case MVT::v8i16: return fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(RetVT, Op0, Op1);
4610 case MVT::v4i32: return fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(RetVT, Op0, Op1);
4611 default: return Register();
4612 }
4613}
4614
4615// FastEmit functions for ISD::AVGCEILU.
4616
4617Register fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4618 if (RetVT.SimpleTy != MVT::v16i8)
4619 return Register();
4620 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
4621}
4622
4623Register fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4624 if (RetVT.SimpleTy != MVT::v8i16)
4625 return Register();
4626 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
4627}
4628
4629Register fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4630 if (RetVT.SimpleTy != MVT::v4i32)
4631 return Register();
4632 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
4633}
4634
4635Register fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4636 switch (VT.SimpleTy) {
4637 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
4638 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
4639 case MVT::v4i32: return fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(RetVT, Op0, Op1);
4640 default: return Register();
4641 }
4642}
4643
4644// FastEmit functions for ISD::AVGFLOORS.
4645
4646Register fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4647 if (RetVT.SimpleTy != MVT::v16i8)
4648 return Register();
4649 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
4650}
4651
4652Register fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4653 if (RetVT.SimpleTy != MVT::v8i16)
4654 return Register();
4655 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
4656}
4657
4658Register fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4659 if (RetVT.SimpleTy != MVT::v4i32)
4660 return Register();
4661 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
4662}
4663
4664Register fastEmit_ISD_AVGFLOORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4665 switch (VT.SimpleTy) {
4666 case MVT::v16i8: return fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(RetVT, Op0, Op1);
4667 case MVT::v8i16: return fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(RetVT, Op0, Op1);
4668 case MVT::v4i32: return fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(RetVT, Op0, Op1);
4669 default: return Register();
4670 }
4671}
4672
4673// FastEmit functions for ISD::AVGFLOORU.
4674
4675Register fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4676 if (RetVT.SimpleTy != MVT::v16i8)
4677 return Register();
4678 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
4679}
4680
4681Register fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4682 if (RetVT.SimpleTy != MVT::v8i16)
4683 return Register();
4684 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
4685}
4686
4687Register fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4688 if (RetVT.SimpleTy != MVT::v4i32)
4689 return Register();
4690 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
4691}
4692
4693Register fastEmit_ISD_AVGFLOORU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4694 switch (VT.SimpleTy) {
4695 case MVT::v16i8: return fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(RetVT, Op0, Op1);
4696 case MVT::v8i16: return fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(RetVT, Op0, Op1);
4697 case MVT::v4i32: return fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(RetVT, Op0, Op1);
4698 default: return Register();
4699 }
4700}
4701
4702// FastEmit functions for ISD::FADD.
4703
4704Register fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
4705 if (RetVT.SimpleTy != MVT::f16)
4706 return Register();
4707 if ((Subtarget->hasFullFP16())) {
4708 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDH, RC: &ARM::HPRRegClass, Op0, Op1);
4709 }
4710 return Register();
4711}
4712
4713Register fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
4714 if (RetVT.SimpleTy != MVT::f32)
4715 return Register();
4716 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
4717 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDS, RC: &ARM::SPRRegClass, Op0, Op1);
4718 }
4719 return Register();
4720}
4721
4722Register fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
4723 if (RetVT.SimpleTy != MVT::f64)
4724 return Register();
4725 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
4726 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDD, RC: &ARM::DPRRegClass, Op0, Op1);
4727 }
4728 return Register();
4729}
4730
4731Register fastEmit_ISD_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
4732 if (RetVT.SimpleTy != MVT::v4f16)
4733 return Register();
4734 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4735 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDhd, RC: &ARM::DPRRegClass, Op0, Op1);
4736 }
4737 return Register();
4738}
4739
4740Register fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
4741 if (RetVT.SimpleTy != MVT::v8f16)
4742 return Register();
4743 if ((Subtarget->hasMVEFloatOps())) {
4744 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDf16, RC: &ARM::MQPRRegClass, Op0, Op1);
4745 }
4746 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4747 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDhq, RC: &ARM::QPRRegClass, Op0, Op1);
4748 }
4749 return Register();
4750}
4751
4752Register fastEmit_ISD_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
4753 if (RetVT.SimpleTy != MVT::v2f32)
4754 return Register();
4755 if ((Subtarget->hasNEON())) {
4756 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDfd, RC: &ARM::DPRRegClass, Op0, Op1);
4757 }
4758 return Register();
4759}
4760
4761Register fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
4762 if (RetVT.SimpleTy != MVT::v4f32)
4763 return Register();
4764 if ((Subtarget->hasMVEFloatOps())) {
4765 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDf32, RC: &ARM::MQPRRegClass, Op0, Op1);
4766 }
4767 if ((Subtarget->hasNEON())) {
4768 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDfq, RC: &ARM::QPRRegClass, Op0, Op1);
4769 }
4770 return Register();
4771}
4772
4773Register fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4774 switch (VT.SimpleTy) {
4775 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
4776 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
4777 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
4778 case MVT::v4f16: return fastEmit_ISD_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
4779 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
4780 case MVT::v2f32: return fastEmit_ISD_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
4781 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
4782 default: return Register();
4783 }
4784}
4785
4786// FastEmit functions for ISD::FDIV.
4787
4788Register fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
4789 if (RetVT.SimpleTy != MVT::f16)
4790 return Register();
4791 if ((Subtarget->hasFullFP16())) {
4792 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVH, RC: &ARM::HPRRegClass, Op0, Op1);
4793 }
4794 return Register();
4795}
4796
4797Register fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
4798 if (RetVT.SimpleTy != MVT::f32)
4799 return Register();
4800 if ((Subtarget->hasVFP2Base())) {
4801 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVS, RC: &ARM::SPRRegClass, Op0, Op1);
4802 }
4803 return Register();
4804}
4805
4806Register fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
4807 if (RetVT.SimpleTy != MVT::f64)
4808 return Register();
4809 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
4810 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVD, RC: &ARM::DPRRegClass, Op0, Op1);
4811 }
4812 return Register();
4813}
4814
4815Register fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4816 switch (VT.SimpleTy) {
4817 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
4818 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
4819 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
4820 default: return Register();
4821 }
4822}
4823
4824// FastEmit functions for ISD::FMAXIMUM.
4825
4826Register fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
4827 if (RetVT.SimpleTy != MVT::v4f16)
4828 return Register();
4829 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4830 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXhd, RC: &ARM::DPRRegClass, Op0, Op1);
4831 }
4832 return Register();
4833}
4834
4835Register fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
4836 if (RetVT.SimpleTy != MVT::v8f16)
4837 return Register();
4838 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4839 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXhq, RC: &ARM::QPRRegClass, Op0, Op1);
4840 }
4841 return Register();
4842}
4843
4844Register fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
4845 if (RetVT.SimpleTy != MVT::v2f32)
4846 return Register();
4847 if ((Subtarget->hasNEON())) {
4848 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXfd, RC: &ARM::DPRRegClass, Op0, Op1);
4849 }
4850 return Register();
4851}
4852
4853Register fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
4854 if (RetVT.SimpleTy != MVT::v4f32)
4855 return Register();
4856 if ((Subtarget->hasNEON())) {
4857 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXfq, RC: &ARM::QPRRegClass, Op0, Op1);
4858 }
4859 return Register();
4860}
4861
4862Register fastEmit_ISD_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4863 switch (VT.SimpleTy) {
4864 case MVT::v4f16: return fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
4865 case MVT::v8f16: return fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
4866 case MVT::v2f32: return fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
4867 case MVT::v4f32: return fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
4868 default: return Register();
4869 }
4870}
4871
4872// FastEmit functions for ISD::FMAXNUM.
4873
4874Register fastEmit_ISD_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
4875 if (RetVT.SimpleTy != MVT::f16)
4876 return Register();
4877 if ((Subtarget->hasFullFP16())) {
4878 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMH, RC: &ARM::HPRRegClass, Op0, Op1);
4879 }
4880 return Register();
4881}
4882
4883Register fastEmit_ISD_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
4884 if (RetVT.SimpleTy != MVT::f32)
4885 return Register();
4886 if ((Subtarget->hasFPARMv8Base())) {
4887 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMS, RC: &ARM::SPRRegClass, Op0, Op1);
4888 }
4889 return Register();
4890}
4891
4892Register fastEmit_ISD_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
4893 if (RetVT.SimpleTy != MVT::f64)
4894 return Register();
4895 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
4896 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMD, RC: &ARM::DPRRegClass, Op0, Op1);
4897 }
4898 return Register();
4899}
4900
4901Register fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
4902 if (RetVT.SimpleTy != MVT::v4f16)
4903 return Register();
4904 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4905 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNDh, RC: &ARM::DPRRegClass, Op0, Op1);
4906 }
4907 return Register();
4908}
4909
4910Register fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
4911 if (RetVT.SimpleTy != MVT::v8f16)
4912 return Register();
4913 if ((Subtarget->hasMVEFloatOps())) {
4914 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXNMf16, RC: &ARM::MQPRRegClass, Op0, Op1);
4915 }
4916 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4917 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNQh, RC: &ARM::QPRRegClass, Op0, Op1);
4918 }
4919 return Register();
4920}
4921
4922Register fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
4923 if (RetVT.SimpleTy != MVT::v2f32)
4924 return Register();
4925 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
4926 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNDf, RC: &ARM::DPRRegClass, Op0, Op1);
4927 }
4928 return Register();
4929}
4930
4931Register fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
4932 if (RetVT.SimpleTy != MVT::v4f32)
4933 return Register();
4934 if ((Subtarget->hasMVEFloatOps())) {
4935 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXNMf32, RC: &ARM::MQPRRegClass, Op0, Op1);
4936 }
4937 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
4938 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNQf, RC: &ARM::QPRRegClass, Op0, Op1);
4939 }
4940 return Register();
4941}
4942
4943Register fastEmit_ISD_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4944 switch (VT.SimpleTy) {
4945 case MVT::f16: return fastEmit_ISD_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
4946 case MVT::f32: return fastEmit_ISD_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
4947 case MVT::f64: return fastEmit_ISD_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
4948 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
4949 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
4950 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
4951 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
4952 default: return Register();
4953 }
4954}
4955
4956// FastEmit functions for ISD::FMINIMUM.
4957
4958Register fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
4959 if (RetVT.SimpleTy != MVT::v4f16)
4960 return Register();
4961 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4962 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINhd, RC: &ARM::DPRRegClass, Op0, Op1);
4963 }
4964 return Register();
4965}
4966
4967Register fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
4968 if (RetVT.SimpleTy != MVT::v8f16)
4969 return Register();
4970 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4971 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINhq, RC: &ARM::QPRRegClass, Op0, Op1);
4972 }
4973 return Register();
4974}
4975
4976Register fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
4977 if (RetVT.SimpleTy != MVT::v2f32)
4978 return Register();
4979 if ((Subtarget->hasNEON())) {
4980 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINfd, RC: &ARM::DPRRegClass, Op0, Op1);
4981 }
4982 return Register();
4983}
4984
4985Register fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
4986 if (RetVT.SimpleTy != MVT::v4f32)
4987 return Register();
4988 if ((Subtarget->hasNEON())) {
4989 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINfq, RC: &ARM::QPRRegClass, Op0, Op1);
4990 }
4991 return Register();
4992}
4993
4994Register fastEmit_ISD_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4995 switch (VT.SimpleTy) {
4996 case MVT::v4f16: return fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
4997 case MVT::v8f16: return fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
4998 case MVT::v2f32: return fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
4999 case MVT::v4f32: return fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
5000 default: return Register();
5001 }
5002}
5003
5004// FastEmit functions for ISD::FMINNUM.
5005
5006Register fastEmit_ISD_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5007 if (RetVT.SimpleTy != MVT::f16)
5008 return Register();
5009 if ((Subtarget->hasFullFP16())) {
5010 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMH, RC: &ARM::HPRRegClass, Op0, Op1);
5011 }
5012 return Register();
5013}
5014
5015Register fastEmit_ISD_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5016 if (RetVT.SimpleTy != MVT::f32)
5017 return Register();
5018 if ((Subtarget->hasFPARMv8Base())) {
5019 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMS, RC: &ARM::SPRRegClass, Op0, Op1);
5020 }
5021 return Register();
5022}
5023
5024Register fastEmit_ISD_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5025 if (RetVT.SimpleTy != MVT::f64)
5026 return Register();
5027 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
5028 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMD, RC: &ARM::DPRRegClass, Op0, Op1);
5029 }
5030 return Register();
5031}
5032
5033Register fastEmit_ISD_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5034 if (RetVT.SimpleTy != MVT::v4f16)
5035 return Register();
5036 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5037 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNDh, RC: &ARM::DPRRegClass, Op0, Op1);
5038 }
5039 return Register();
5040}
5041
5042Register fastEmit_ISD_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5043 if (RetVT.SimpleTy != MVT::v8f16)
5044 return Register();
5045 if ((Subtarget->hasMVEFloatOps())) {
5046 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINNMf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5047 }
5048 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5049 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNQh, RC: &ARM::QPRRegClass, Op0, Op1);
5050 }
5051 return Register();
5052}
5053
5054Register fastEmit_ISD_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5055 if (RetVT.SimpleTy != MVT::v2f32)
5056 return Register();
5057 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
5058 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNDf, RC: &ARM::DPRRegClass, Op0, Op1);
5059 }
5060 return Register();
5061}
5062
5063Register fastEmit_ISD_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5064 if (RetVT.SimpleTy != MVT::v4f32)
5065 return Register();
5066 if ((Subtarget->hasMVEFloatOps())) {
5067 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINNMf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5068 }
5069 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
5070 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNQf, RC: &ARM::QPRRegClass, Op0, Op1);
5071 }
5072 return Register();
5073}
5074
5075Register fastEmit_ISD_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5076 switch (VT.SimpleTy) {
5077 case MVT::f16: return fastEmit_ISD_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
5078 case MVT::f32: return fastEmit_ISD_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
5079 case MVT::f64: return fastEmit_ISD_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
5080 case MVT::v4f16: return fastEmit_ISD_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
5081 case MVT::v8f16: return fastEmit_ISD_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
5082 case MVT::v2f32: return fastEmit_ISD_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
5083 case MVT::v4f32: return fastEmit_ISD_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
5084 default: return Register();
5085 }
5086}
5087
5088// FastEmit functions for ISD::FMUL.
5089
5090Register fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5091 if (RetVT.SimpleTy != MVT::f16)
5092 return Register();
5093 if ((Subtarget->hasFullFP16())) {
5094 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULH, RC: &ARM::HPRRegClass, Op0, Op1);
5095 }
5096 return Register();
5097}
5098
5099Register fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5100 if (RetVT.SimpleTy != MVT::f32)
5101 return Register();
5102 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
5103 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULS, RC: &ARM::SPRRegClass, Op0, Op1);
5104 }
5105 return Register();
5106}
5107
5108Register fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5109 if (RetVT.SimpleTy != MVT::f64)
5110 return Register();
5111 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
5112 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULD, RC: &ARM::DPRRegClass, Op0, Op1);
5113 }
5114 return Register();
5115}
5116
5117Register fastEmit_ISD_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5118 if (RetVT.SimpleTy != MVT::v4f16)
5119 return Register();
5120 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5121 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULhd, RC: &ARM::DPRRegClass, Op0, Op1);
5122 }
5123 return Register();
5124}
5125
5126Register fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5127 if (RetVT.SimpleTy != MVT::v8f16)
5128 return Register();
5129 if ((Subtarget->hasMVEFloatOps())) {
5130 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5131 }
5132 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5133 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULhq, RC: &ARM::QPRRegClass, Op0, Op1);
5134 }
5135 return Register();
5136}
5137
5138Register fastEmit_ISD_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5139 if (RetVT.SimpleTy != MVT::v2f32)
5140 return Register();
5141 if ((Subtarget->hasNEON())) {
5142 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULfd, RC: &ARM::DPRRegClass, Op0, Op1);
5143 }
5144 return Register();
5145}
5146
5147Register fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5148 if (RetVT.SimpleTy != MVT::v4f32)
5149 return Register();
5150 if ((Subtarget->hasMVEFloatOps())) {
5151 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5152 }
5153 if ((Subtarget->hasNEON())) {
5154 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULfq, RC: &ARM::QPRRegClass, Op0, Op1);
5155 }
5156 return Register();
5157}
5158
5159Register fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5160 switch (VT.SimpleTy) {
5161 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
5162 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
5163 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
5164 case MVT::v4f16: return fastEmit_ISD_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
5165 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
5166 case MVT::v2f32: return fastEmit_ISD_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
5167 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
5168 default: return Register();
5169 }
5170}
5171
5172// FastEmit functions for ISD::FSUB.
5173
5174Register fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5175 if (RetVT.SimpleTy != MVT::f16)
5176 return Register();
5177 if ((Subtarget->hasFullFP16())) {
5178 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBH, RC: &ARM::HPRRegClass, Op0, Op1);
5179 }
5180 return Register();
5181}
5182
5183Register fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5184 if (RetVT.SimpleTy != MVT::f32)
5185 return Register();
5186 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
5187 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBS, RC: &ARM::SPRRegClass, Op0, Op1);
5188 }
5189 return Register();
5190}
5191
5192Register fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5193 if (RetVT.SimpleTy != MVT::f64)
5194 return Register();
5195 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
5196 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBD, RC: &ARM::DPRRegClass, Op0, Op1);
5197 }
5198 return Register();
5199}
5200
5201Register fastEmit_ISD_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5202 if (RetVT.SimpleTy != MVT::v4f16)
5203 return Register();
5204 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5205 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBhd, RC: &ARM::DPRRegClass, Op0, Op1);
5206 }
5207 return Register();
5208}
5209
5210Register fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5211 if (RetVT.SimpleTy != MVT::v8f16)
5212 return Register();
5213 if ((Subtarget->hasMVEFloatOps())) {
5214 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5215 }
5216 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5217 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBhq, RC: &ARM::QPRRegClass, Op0, Op1);
5218 }
5219 return Register();
5220}
5221
5222Register fastEmit_ISD_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5223 if (RetVT.SimpleTy != MVT::v2f32)
5224 return Register();
5225 if ((Subtarget->hasNEON())) {
5226 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBfd, RC: &ARM::DPRRegClass, Op0, Op1);
5227 }
5228 return Register();
5229}
5230
5231Register fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5232 if (RetVT.SimpleTy != MVT::v4f32)
5233 return Register();
5234 if ((Subtarget->hasMVEFloatOps())) {
5235 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5236 }
5237 if ((Subtarget->hasNEON())) {
5238 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBfq, RC: &ARM::QPRRegClass, Op0, Op1);
5239 }
5240 return Register();
5241}
5242
5243Register fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5244 switch (VT.SimpleTy) {
5245 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
5246 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
5247 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
5248 case MVT::v4f16: return fastEmit_ISD_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
5249 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
5250 case MVT::v2f32: return fastEmit_ISD_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
5251 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
5252 default: return Register();
5253 }
5254}
5255
5256// FastEmit functions for ISD::MUL.
5257
5258Register fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5259 if (RetVT.SimpleTy != MVT::i32)
5260 return Register();
5261 if ((Subtarget->isThumb2())) {
5262 return fastEmitInst_rr(MachineInstOpcode: ARM::t2MUL, RC: &ARM::rGPRRegClass, Op0, Op1);
5263 }
5264 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5265 return fastEmitInst_rr(MachineInstOpcode: ARM::tMUL, RC: &ARM::tGPRRegClass, Op0, Op1);
5266 }
5267 if ((!Subtarget->isThumb()) && (!Subtarget->hasV6Ops()) && (Subtarget->useMulOps())) {
5268 return fastEmitInst_rr(MachineInstOpcode: ARM::MULv5, RC: &ARM::GPRnopcRegClass, Op0, Op1);
5269 }
5270 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
5271 return fastEmitInst_rr(MachineInstOpcode: ARM::MUL, RC: &ARM::GPRnopcRegClass, Op0, Op1);
5272 }
5273 return Register();
5274}
5275
5276Register fastEmit_ISD_MUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
5277 if (RetVT.SimpleTy != MVT::v8i8)
5278 return Register();
5279 if ((Subtarget->hasNEON())) {
5280 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
5281 }
5282 return Register();
5283}
5284
5285Register fastEmit_ISD_MUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5286 if (RetVT.SimpleTy != MVT::v16i8)
5287 return Register();
5288 if ((Subtarget->hasMVEIntegerOps())) {
5289 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULi8, RC: &ARM::MQPRRegClass, Op0, Op1);
5290 }
5291 if ((Subtarget->hasNEON())) {
5292 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
5293 }
5294 return Register();
5295}
5296
5297Register fastEmit_ISD_MUL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
5298 if (RetVT.SimpleTy != MVT::v4i16)
5299 return Register();
5300 if ((Subtarget->hasNEON())) {
5301 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
5302 }
5303 return Register();
5304}
5305
5306Register fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5307 if (RetVT.SimpleTy != MVT::v8i16)
5308 return Register();
5309 if ((Subtarget->hasMVEIntegerOps())) {
5310 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULi16, RC: &ARM::MQPRRegClass, Op0, Op1);
5311 }
5312 if ((Subtarget->hasNEON())) {
5313 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
5314 }
5315 return Register();
5316}
5317
5318Register fastEmit_ISD_MUL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
5319 if (RetVT.SimpleTy != MVT::v2i32)
5320 return Register();
5321 if ((Subtarget->hasNEON())) {
5322 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
5323 }
5324 return Register();
5325}
5326
5327Register fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5328 if (RetVT.SimpleTy != MVT::v4i32)
5329 return Register();
5330 if ((Subtarget->hasMVEIntegerOps())) {
5331 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULi32, RC: &ARM::MQPRRegClass, Op0, Op1);
5332 }
5333 if ((Subtarget->hasNEON())) {
5334 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
5335 }
5336 return Register();
5337}
5338
5339Register fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5340 switch (VT.SimpleTy) {
5341 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op1);
5342 case MVT::v8i8: return fastEmit_ISD_MUL_MVT_v8i8_rr(RetVT, Op0, Op1);
5343 case MVT::v16i8: return fastEmit_ISD_MUL_MVT_v16i8_rr(RetVT, Op0, Op1);
5344 case MVT::v4i16: return fastEmit_ISD_MUL_MVT_v4i16_rr(RetVT, Op0, Op1);
5345 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
5346 case MVT::v2i32: return fastEmit_ISD_MUL_MVT_v2i32_rr(RetVT, Op0, Op1);
5347 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
5348 default: return Register();
5349 }
5350}
5351
5352// FastEmit functions for ISD::MULHS.
5353
5354Register fastEmit_ISD_MULHS_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5355 if (RetVT.SimpleTy != MVT::i32)
5356 return Register();
5357 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
5358 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SMMUL, RC: &ARM::rGPRRegClass, Op0, Op1);
5359 }
5360 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
5361 return fastEmitInst_rr(MachineInstOpcode: ARM::SMMUL, RC: &ARM::GPRRegClass, Op0, Op1);
5362 }
5363 return Register();
5364}
5365
5366Register fastEmit_ISD_MULHS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5367 if (RetVT.SimpleTy != MVT::v16i8)
5368 return Register();
5369 if ((Subtarget->hasMVEIntegerOps())) {
5370 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHs8, RC: &ARM::MQPRRegClass, Op0, Op1);
5371 }
5372 return Register();
5373}
5374
5375Register fastEmit_ISD_MULHS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5376 if (RetVT.SimpleTy != MVT::v8i16)
5377 return Register();
5378 if ((Subtarget->hasMVEIntegerOps())) {
5379 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHs16, RC: &ARM::MQPRRegClass, Op0, Op1);
5380 }
5381 return Register();
5382}
5383
5384Register fastEmit_ISD_MULHS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5385 if (RetVT.SimpleTy != MVT::v4i32)
5386 return Register();
5387 if ((Subtarget->hasMVEIntegerOps())) {
5388 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHs32, RC: &ARM::MQPRRegClass, Op0, Op1);
5389 }
5390 return Register();
5391}
5392
5393Register fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5394 switch (VT.SimpleTy) {
5395 case MVT::i32: return fastEmit_ISD_MULHS_MVT_i32_rr(RetVT, Op0, Op1);
5396 case MVT::v16i8: return fastEmit_ISD_MULHS_MVT_v16i8_rr(RetVT, Op0, Op1);
5397 case MVT::v8i16: return fastEmit_ISD_MULHS_MVT_v8i16_rr(RetVT, Op0, Op1);
5398 case MVT::v4i32: return fastEmit_ISD_MULHS_MVT_v4i32_rr(RetVT, Op0, Op1);
5399 default: return Register();
5400 }
5401}
5402
5403// FastEmit functions for ISD::MULHU.
5404
5405Register fastEmit_ISD_MULHU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5406 if (RetVT.SimpleTy != MVT::v16i8)
5407 return Register();
5408 if ((Subtarget->hasMVEIntegerOps())) {
5409 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHu8, RC: &ARM::MQPRRegClass, Op0, Op1);
5410 }
5411 return Register();
5412}
5413
5414Register fastEmit_ISD_MULHU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5415 if (RetVT.SimpleTy != MVT::v8i16)
5416 return Register();
5417 if ((Subtarget->hasMVEIntegerOps())) {
5418 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHu16, RC: &ARM::MQPRRegClass, Op0, Op1);
5419 }
5420 return Register();
5421}
5422
5423Register fastEmit_ISD_MULHU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5424 if (RetVT.SimpleTy != MVT::v4i32)
5425 return Register();
5426 if ((Subtarget->hasMVEIntegerOps())) {
5427 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHu32, RC: &ARM::MQPRRegClass, Op0, Op1);
5428 }
5429 return Register();
5430}
5431
5432Register fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5433 switch (VT.SimpleTy) {
5434 case MVT::v16i8: return fastEmit_ISD_MULHU_MVT_v16i8_rr(RetVT, Op0, Op1);
5435 case MVT::v8i16: return fastEmit_ISD_MULHU_MVT_v8i16_rr(RetVT, Op0, Op1);
5436 case MVT::v4i32: return fastEmit_ISD_MULHU_MVT_v4i32_rr(RetVT, Op0, Op1);
5437 default: return Register();
5438 }
5439}
5440
5441// FastEmit functions for ISD::OR.
5442
5443Register fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5444 if (RetVT.SimpleTy != MVT::i32)
5445 return Register();
5446 if ((Subtarget->isThumb2())) {
5447 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ORRrr, RC: &ARM::rGPRRegClass, Op0, Op1);
5448 }
5449 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5450 return fastEmitInst_rr(MachineInstOpcode: ARM::tORR, RC: &ARM::tGPRRegClass, Op0, Op1);
5451 }
5452 if ((!Subtarget->isThumb())) {
5453 return fastEmitInst_rr(MachineInstOpcode: ARM::ORRrr, RC: &ARM::GPRRegClass, Op0, Op1);
5454 }
5455 return Register();
5456}
5457
5458Register fastEmit_ISD_OR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
5459 if (RetVT.SimpleTy != MVT::v8i8)
5460 return Register();
5461 if ((Subtarget->hasNEON())) {
5462 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
5463 }
5464 return Register();
5465}
5466
5467Register fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5468 if (RetVT.SimpleTy != MVT::v16i8)
5469 return Register();
5470 if ((Subtarget->hasMVEIntegerOps())) {
5471 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
5472 }
5473 if ((Subtarget->hasNEON())) {
5474 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
5475 }
5476 return Register();
5477}
5478
5479Register fastEmit_ISD_OR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
5480 if (RetVT.SimpleTy != MVT::v4i16)
5481 return Register();
5482 if ((Subtarget->hasNEON())) {
5483 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
5484 }
5485 return Register();
5486}
5487
5488Register fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5489 if (RetVT.SimpleTy != MVT::v8i16)
5490 return Register();
5491 if ((Subtarget->hasMVEIntegerOps())) {
5492 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
5493 }
5494 if ((Subtarget->hasNEON())) {
5495 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
5496 }
5497 return Register();
5498}
5499
5500Register fastEmit_ISD_OR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
5501 if (RetVT.SimpleTy != MVT::v2i32)
5502 return Register();
5503 if ((Subtarget->hasNEON())) {
5504 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
5505 }
5506 return Register();
5507}
5508
5509Register fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5510 if (RetVT.SimpleTy != MVT::v4i32)
5511 return Register();
5512 if ((Subtarget->hasMVEIntegerOps())) {
5513 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
5514 }
5515 if ((Subtarget->hasNEON())) {
5516 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
5517 }
5518 return Register();
5519}
5520
5521Register fastEmit_ISD_OR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
5522 if (RetVT.SimpleTy != MVT::v1i64)
5523 return Register();
5524 if ((Subtarget->hasNEON())) {
5525 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
5526 }
5527 return Register();
5528}
5529
5530Register fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
5531 if (RetVT.SimpleTy != MVT::v2i64)
5532 return Register();
5533 if ((Subtarget->hasMVEIntegerOps())) {
5534 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
5535 }
5536 if ((Subtarget->hasNEON())) {
5537 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
5538 }
5539 return Register();
5540}
5541
5542Register fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5543 switch (VT.SimpleTy) {
5544 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
5545 case MVT::v8i8: return fastEmit_ISD_OR_MVT_v8i8_rr(RetVT, Op0, Op1);
5546 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
5547 case MVT::v4i16: return fastEmit_ISD_OR_MVT_v4i16_rr(RetVT, Op0, Op1);
5548 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
5549 case MVT::v2i32: return fastEmit_ISD_OR_MVT_v2i32_rr(RetVT, Op0, Op1);
5550 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
5551 case MVT::v1i64: return fastEmit_ISD_OR_MVT_v1i64_rr(RetVT, Op0, Op1);
5552 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
5553 default: return Register();
5554 }
5555}
5556
5557// FastEmit functions for ISD::ROTR.
5558
5559Register fastEmit_ISD_ROTR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5560 if (RetVT.SimpleTy != MVT::i32)
5561 return Register();
5562 if ((Subtarget->isThumb2())) {
5563 return fastEmitInst_rr(MachineInstOpcode: ARM::t2RORrr, RC: &ARM::rGPRRegClass, Op0, Op1);
5564 }
5565 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5566 return fastEmitInst_rr(MachineInstOpcode: ARM::tROR, RC: &ARM::tGPRRegClass, Op0, Op1);
5567 }
5568 return Register();
5569}
5570
5571Register fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5572 switch (VT.SimpleTy) {
5573 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_rr(RetVT, Op0, Op1);
5574 default: return Register();
5575 }
5576}
5577
5578// FastEmit functions for ISD::SADDSAT.
5579
5580Register fastEmit_ISD_SADDSAT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5581 if (RetVT.SimpleTy != MVT::i32)
5582 return Register();
5583 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
5584 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QADD, RC: &ARM::rGPRRegClass, Op0, Op1);
5585 }
5586 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
5587 return fastEmitInst_rr(MachineInstOpcode: ARM::QADD, RC: &ARM::GPRnopcRegClass, Op0, Op1);
5588 }
5589 return Register();
5590}
5591
5592Register fastEmit_ISD_SADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
5593 if (RetVT.SimpleTy != MVT::v8i8)
5594 return Register();
5595 if ((Subtarget->hasNEON())) {
5596 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
5597 }
5598 return Register();
5599}
5600
5601Register fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5602 if (RetVT.SimpleTy != MVT::v16i8)
5603 return Register();
5604 if ((Subtarget->hasMVEIntegerOps())) {
5605 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
5606 }
5607 if ((Subtarget->hasNEON())) {
5608 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
5609 }
5610 return Register();
5611}
5612
5613Register fastEmit_ISD_SADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
5614 if (RetVT.SimpleTy != MVT::v4i16)
5615 return Register();
5616 if ((Subtarget->hasNEON())) {
5617 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
5618 }
5619 return Register();
5620}
5621
5622Register fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5623 if (RetVT.SimpleTy != MVT::v8i16)
5624 return Register();
5625 if ((Subtarget->hasMVEIntegerOps())) {
5626 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
5627 }
5628 if ((Subtarget->hasNEON())) {
5629 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
5630 }
5631 return Register();
5632}
5633
5634Register fastEmit_ISD_SADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
5635 if (RetVT.SimpleTy != MVT::v2i32)
5636 return Register();
5637 if ((Subtarget->hasNEON())) {
5638 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
5639 }
5640 return Register();
5641}
5642
5643Register fastEmit_ISD_SADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5644 if (RetVT.SimpleTy != MVT::v4i32)
5645 return Register();
5646 if ((Subtarget->hasMVEIntegerOps())) {
5647 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
5648 }
5649 if ((Subtarget->hasNEON())) {
5650 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
5651 }
5652 return Register();
5653}
5654
5655Register fastEmit_ISD_SADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
5656 if (RetVT.SimpleTy != MVT::v1i64)
5657 return Register();
5658 if ((Subtarget->hasNEON())) {
5659 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
5660 }
5661 return Register();
5662}
5663
5664Register fastEmit_ISD_SADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
5665 if (RetVT.SimpleTy != MVT::v2i64)
5666 return Register();
5667 if ((Subtarget->hasNEON())) {
5668 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
5669 }
5670 return Register();
5671}
5672
5673Register fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5674 switch (VT.SimpleTy) {
5675 case MVT::i32: return fastEmit_ISD_SADDSAT_MVT_i32_rr(RetVT, Op0, Op1);
5676 case MVT::v8i8: return fastEmit_ISD_SADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
5677 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
5678 case MVT::v4i16: return fastEmit_ISD_SADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
5679 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
5680 case MVT::v2i32: return fastEmit_ISD_SADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
5681 case MVT::v4i32: return fastEmit_ISD_SADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
5682 case MVT::v1i64: return fastEmit_ISD_SADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
5683 case MVT::v2i64: return fastEmit_ISD_SADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
5684 default: return Register();
5685 }
5686}
5687
5688// FastEmit functions for ISD::SDIV.
5689
5690Register fastEmit_ISD_SDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5691 if (RetVT.SimpleTy != MVT::i32)
5692 return Register();
5693 if ((Subtarget->hasDivideInThumbMode()) && (Subtarget->hasV8MBaselineOps()) && (Subtarget->isThumb())) {
5694 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SDIV, RC: &ARM::rGPRRegClass, Op0, Op1);
5695 }
5696 if ((Subtarget->hasDivideInARMMode()) && (!Subtarget->isThumb())) {
5697 return fastEmitInst_rr(MachineInstOpcode: ARM::SDIV, RC: &ARM::GPRRegClass, Op0, Op1);
5698 }
5699 return Register();
5700}
5701
5702Register fastEmit_ISD_SDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5703 switch (VT.SimpleTy) {
5704 case MVT::i32: return fastEmit_ISD_SDIV_MVT_i32_rr(RetVT, Op0, Op1);
5705 default: return Register();
5706 }
5707}
5708
5709// FastEmit functions for ISD::SHL.
5710
5711Register fastEmit_ISD_SHL_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5712 if (RetVT.SimpleTy != MVT::i32)
5713 return Register();
5714 if ((Subtarget->isThumb2())) {
5715 return fastEmitInst_rr(MachineInstOpcode: ARM::t2LSLrr, RC: &ARM::rGPRRegClass, Op0, Op1);
5716 }
5717 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5718 return fastEmitInst_rr(MachineInstOpcode: ARM::tLSLrr, RC: &ARM::tGPRRegClass, Op0, Op1);
5719 }
5720 return Register();
5721}
5722
5723Register fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5724 switch (VT.SimpleTy) {
5725 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_rr(RetVT, Op0, Op1);
5726 default: return Register();
5727 }
5728}
5729
5730// FastEmit functions for ISD::SMAX.
5731
5732Register fastEmit_ISD_SMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
5733 if (RetVT.SimpleTy != MVT::v8i8)
5734 return Register();
5735 if ((Subtarget->hasNEON())) {
5736 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
5737 }
5738 return Register();
5739}
5740
5741Register fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5742 if (RetVT.SimpleTy != MVT::v16i8)
5743 return Register();
5744 if ((Subtarget->hasMVEIntegerOps())) {
5745 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXs8, RC: &ARM::MQPRRegClass, Op0, Op1);
5746 }
5747 if ((Subtarget->hasNEON())) {
5748 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
5749 }
5750 return Register();
5751}
5752
5753Register fastEmit_ISD_SMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
5754 if (RetVT.SimpleTy != MVT::v4i16)
5755 return Register();
5756 if ((Subtarget->hasNEON())) {
5757 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
5758 }
5759 return Register();
5760}
5761
5762Register fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5763 if (RetVT.SimpleTy != MVT::v8i16)
5764 return Register();
5765 if ((Subtarget->hasMVEIntegerOps())) {
5766 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXs16, RC: &ARM::MQPRRegClass, Op0, Op1);
5767 }
5768 if ((Subtarget->hasNEON())) {
5769 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
5770 }
5771 return Register();
5772}
5773
5774Register fastEmit_ISD_SMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
5775 if (RetVT.SimpleTy != MVT::v2i32)
5776 return Register();
5777 if ((Subtarget->hasNEON())) {
5778 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
5779 }
5780 return Register();
5781}
5782
5783Register fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5784 if (RetVT.SimpleTy != MVT::v4i32)
5785 return Register();
5786 if ((Subtarget->hasMVEIntegerOps())) {
5787 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXs32, RC: &ARM::MQPRRegClass, Op0, Op1);
5788 }
5789 if ((Subtarget->hasNEON())) {
5790 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
5791 }
5792 return Register();
5793}
5794
5795Register fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5796 switch (VT.SimpleTy) {
5797 case MVT::v8i8: return fastEmit_ISD_SMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
5798 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
5799 case MVT::v4i16: return fastEmit_ISD_SMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
5800 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
5801 case MVT::v2i32: return fastEmit_ISD_SMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
5802 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
5803 default: return Register();
5804 }
5805}
5806
5807// FastEmit functions for ISD::SMIN.
5808
5809Register fastEmit_ISD_SMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
5810 if (RetVT.SimpleTy != MVT::v8i8)
5811 return Register();
5812 if ((Subtarget->hasNEON())) {
5813 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
5814 }
5815 return Register();
5816}
5817
5818Register fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5819 if (RetVT.SimpleTy != MVT::v16i8)
5820 return Register();
5821 if ((Subtarget->hasMVEIntegerOps())) {
5822 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINs8, RC: &ARM::MQPRRegClass, Op0, Op1);
5823 }
5824 if ((Subtarget->hasNEON())) {
5825 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
5826 }
5827 return Register();
5828}
5829
5830Register fastEmit_ISD_SMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
5831 if (RetVT.SimpleTy != MVT::v4i16)
5832 return Register();
5833 if ((Subtarget->hasNEON())) {
5834 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
5835 }
5836 return Register();
5837}
5838
5839Register fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5840 if (RetVT.SimpleTy != MVT::v8i16)
5841 return Register();
5842 if ((Subtarget->hasMVEIntegerOps())) {
5843 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINs16, RC: &ARM::MQPRRegClass, Op0, Op1);
5844 }
5845 if ((Subtarget->hasNEON())) {
5846 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
5847 }
5848 return Register();
5849}
5850
5851Register fastEmit_ISD_SMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
5852 if (RetVT.SimpleTy != MVT::v2i32)
5853 return Register();
5854 if ((Subtarget->hasNEON())) {
5855 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
5856 }
5857 return Register();
5858}
5859
5860Register fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5861 if (RetVT.SimpleTy != MVT::v4i32)
5862 return Register();
5863 if ((Subtarget->hasMVEIntegerOps())) {
5864 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINs32, RC: &ARM::MQPRRegClass, Op0, Op1);
5865 }
5866 if ((Subtarget->hasNEON())) {
5867 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
5868 }
5869 return Register();
5870}
5871
5872Register fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5873 switch (VT.SimpleTy) {
5874 case MVT::v8i8: return fastEmit_ISD_SMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
5875 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
5876 case MVT::v4i16: return fastEmit_ISD_SMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
5877 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
5878 case MVT::v2i32: return fastEmit_ISD_SMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
5879 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
5880 default: return Register();
5881 }
5882}
5883
5884// FastEmit functions for ISD::SRA.
5885
5886Register fastEmit_ISD_SRA_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5887 if (RetVT.SimpleTy != MVT::i32)
5888 return Register();
5889 if ((Subtarget->isThumb2())) {
5890 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ASRrr, RC: &ARM::rGPRRegClass, Op0, Op1);
5891 }
5892 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5893 return fastEmitInst_rr(MachineInstOpcode: ARM::tASRrr, RC: &ARM::tGPRRegClass, Op0, Op1);
5894 }
5895 return Register();
5896}
5897
5898Register fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5899 switch (VT.SimpleTy) {
5900 case MVT::i32: return fastEmit_ISD_SRA_MVT_i32_rr(RetVT, Op0, Op1);
5901 default: return Register();
5902 }
5903}
5904
5905// FastEmit functions for ISD::SRL.
5906
5907Register fastEmit_ISD_SRL_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5908 if (RetVT.SimpleTy != MVT::i32)
5909 return Register();
5910 if ((Subtarget->isThumb2())) {
5911 return fastEmitInst_rr(MachineInstOpcode: ARM::t2LSRrr, RC: &ARM::rGPRRegClass, Op0, Op1);
5912 }
5913 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5914 return fastEmitInst_rr(MachineInstOpcode: ARM::tLSRrr, RC: &ARM::tGPRRegClass, Op0, Op1);
5915 }
5916 return Register();
5917}
5918
5919Register fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5920 switch (VT.SimpleTy) {
5921 case MVT::i32: return fastEmit_ISD_SRL_MVT_i32_rr(RetVT, Op0, Op1);
5922 default: return Register();
5923 }
5924}
5925
5926// FastEmit functions for ISD::SSUBSAT.
5927
5928Register fastEmit_ISD_SSUBSAT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5929 if (RetVT.SimpleTy != MVT::i32)
5930 return Register();
5931 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
5932 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QSUB, RC: &ARM::rGPRRegClass, Op0, Op1);
5933 }
5934 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
5935 return fastEmitInst_rr(MachineInstOpcode: ARM::QSUB, RC: &ARM::GPRnopcRegClass, Op0, Op1);
5936 }
5937 return Register();
5938}
5939
5940Register fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
5941 if (RetVT.SimpleTy != MVT::v8i8)
5942 return Register();
5943 if ((Subtarget->hasNEON())) {
5944 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
5945 }
5946 return Register();
5947}
5948
5949Register fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5950 if (RetVT.SimpleTy != MVT::v16i8)
5951 return Register();
5952 if ((Subtarget->hasMVEIntegerOps())) {
5953 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBs8, RC: &ARM::MQPRRegClass, Op0, Op1);
5954 }
5955 if ((Subtarget->hasNEON())) {
5956 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
5957 }
5958 return Register();
5959}
5960
5961Register fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
5962 if (RetVT.SimpleTy != MVT::v4i16)
5963 return Register();
5964 if ((Subtarget->hasNEON())) {
5965 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
5966 }
5967 return Register();
5968}
5969
5970Register fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5971 if (RetVT.SimpleTy != MVT::v8i16)
5972 return Register();
5973 if ((Subtarget->hasMVEIntegerOps())) {
5974 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBs16, RC: &ARM::MQPRRegClass, Op0, Op1);
5975 }
5976 if ((Subtarget->hasNEON())) {
5977 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
5978 }
5979 return Register();
5980}
5981
5982Register fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
5983 if (RetVT.SimpleTy != MVT::v2i32)
5984 return Register();
5985 if ((Subtarget->hasNEON())) {
5986 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
5987 }
5988 return Register();
5989}
5990
5991Register fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5992 if (RetVT.SimpleTy != MVT::v4i32)
5993 return Register();
5994 if ((Subtarget->hasMVEIntegerOps())) {
5995 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBs32, RC: &ARM::MQPRRegClass, Op0, Op1);
5996 }
5997 if ((Subtarget->hasNEON())) {
5998 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
5999 }
6000 return Register();
6001}
6002
6003Register fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
6004 if (RetVT.SimpleTy != MVT::v1i64)
6005 return Register();
6006 if ((Subtarget->hasNEON())) {
6007 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
6008 }
6009 return Register();
6010}
6011
6012Register fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6013 if (RetVT.SimpleTy != MVT::v2i64)
6014 return Register();
6015 if ((Subtarget->hasNEON())) {
6016 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
6017 }
6018 return Register();
6019}
6020
6021Register fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6022 switch (VT.SimpleTy) {
6023 case MVT::i32: return fastEmit_ISD_SSUBSAT_MVT_i32_rr(RetVT, Op0, Op1);
6024 case MVT::v8i8: return fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
6025 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
6026 case MVT::v4i16: return fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
6027 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
6028 case MVT::v2i32: return fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
6029 case MVT::v4i32: return fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
6030 case MVT::v1i64: return fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
6031 case MVT::v2i64: return fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
6032 default: return Register();
6033 }
6034}
6035
6036// FastEmit functions for ISD::SUB.
6037
6038Register fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6039 if (RetVT.SimpleTy != MVT::i32)
6040 return Register();
6041 if ((Subtarget->isThumb2())) {
6042 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SUBrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
6043 }
6044 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6045 return fastEmitInst_rr(MachineInstOpcode: ARM::tSUBrr, RC: &ARM::tGPRRegClass, Op0, Op1);
6046 }
6047 if ((!Subtarget->isThumb())) {
6048 return fastEmitInst_rr(MachineInstOpcode: ARM::SUBrr, RC: &ARM::GPRRegClass, Op0, Op1);
6049 }
6050 return Register();
6051}
6052
6053Register fastEmit_ISD_SUB_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6054 if (RetVT.SimpleTy != MVT::v8i8)
6055 return Register();
6056 if ((Subtarget->hasNEON())) {
6057 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6058 }
6059 return Register();
6060}
6061
6062Register fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6063 if (RetVT.SimpleTy != MVT::v16i8)
6064 return Register();
6065 if ((Subtarget->hasMVEIntegerOps())) {
6066 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBi8, RC: &ARM::MQPRRegClass, Op0, Op1);
6067 }
6068 if ((Subtarget->hasNEON())) {
6069 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6070 }
6071 return Register();
6072}
6073
6074Register fastEmit_ISD_SUB_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6075 if (RetVT.SimpleTy != MVT::v4i16)
6076 return Register();
6077 if ((Subtarget->hasNEON())) {
6078 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6079 }
6080 return Register();
6081}
6082
6083Register fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6084 if (RetVT.SimpleTy != MVT::v8i16)
6085 return Register();
6086 if ((Subtarget->hasMVEIntegerOps())) {
6087 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBi16, RC: &ARM::MQPRRegClass, Op0, Op1);
6088 }
6089 if ((Subtarget->hasNEON())) {
6090 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6091 }
6092 return Register();
6093}
6094
6095Register fastEmit_ISD_SUB_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6096 if (RetVT.SimpleTy != MVT::v2i32)
6097 return Register();
6098 if ((Subtarget->hasNEON())) {
6099 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6100 }
6101 return Register();
6102}
6103
6104Register fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6105 if (RetVT.SimpleTy != MVT::v4i32)
6106 return Register();
6107 if ((Subtarget->hasMVEIntegerOps())) {
6108 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBi32, RC: &ARM::MQPRRegClass, Op0, Op1);
6109 }
6110 if ((Subtarget->hasNEON())) {
6111 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6112 }
6113 return Register();
6114}
6115
6116Register fastEmit_ISD_SUB_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
6117 if (RetVT.SimpleTy != MVT::v1i64)
6118 return Register();
6119 if ((Subtarget->hasNEON())) {
6120 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
6121 }
6122 return Register();
6123}
6124
6125Register fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6126 if (RetVT.SimpleTy != MVT::v2i64)
6127 return Register();
6128 if ((Subtarget->hasNEON())) {
6129 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
6130 }
6131 return Register();
6132}
6133
6134Register fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6135 switch (VT.SimpleTy) {
6136 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
6137 case MVT::v8i8: return fastEmit_ISD_SUB_MVT_v8i8_rr(RetVT, Op0, Op1);
6138 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
6139 case MVT::v4i16: return fastEmit_ISD_SUB_MVT_v4i16_rr(RetVT, Op0, Op1);
6140 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
6141 case MVT::v2i32: return fastEmit_ISD_SUB_MVT_v2i32_rr(RetVT, Op0, Op1);
6142 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
6143 case MVT::v1i64: return fastEmit_ISD_SUB_MVT_v1i64_rr(RetVT, Op0, Op1);
6144 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
6145 default: return Register();
6146 }
6147}
6148
6149// FastEmit functions for ISD::UADDSAT.
6150
6151Register fastEmit_ISD_UADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6152 if (RetVT.SimpleTy != MVT::v8i8)
6153 return Register();
6154 if ((Subtarget->hasNEON())) {
6155 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6156 }
6157 return Register();
6158}
6159
6160Register fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6161 if (RetVT.SimpleTy != MVT::v16i8)
6162 return Register();
6163 if ((Subtarget->hasMVEIntegerOps())) {
6164 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
6165 }
6166 if ((Subtarget->hasNEON())) {
6167 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6168 }
6169 return Register();
6170}
6171
6172Register fastEmit_ISD_UADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6173 if (RetVT.SimpleTy != MVT::v4i16)
6174 return Register();
6175 if ((Subtarget->hasNEON())) {
6176 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6177 }
6178 return Register();
6179}
6180
6181Register fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6182 if (RetVT.SimpleTy != MVT::v8i16)
6183 return Register();
6184 if ((Subtarget->hasMVEIntegerOps())) {
6185 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
6186 }
6187 if ((Subtarget->hasNEON())) {
6188 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6189 }
6190 return Register();
6191}
6192
6193Register fastEmit_ISD_UADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6194 if (RetVT.SimpleTy != MVT::v2i32)
6195 return Register();
6196 if ((Subtarget->hasNEON())) {
6197 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6198 }
6199 return Register();
6200}
6201
6202Register fastEmit_ISD_UADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6203 if (RetVT.SimpleTy != MVT::v4i32)
6204 return Register();
6205 if ((Subtarget->hasMVEIntegerOps())) {
6206 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
6207 }
6208 if ((Subtarget->hasNEON())) {
6209 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6210 }
6211 return Register();
6212}
6213
6214Register fastEmit_ISD_UADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
6215 if (RetVT.SimpleTy != MVT::v1i64)
6216 return Register();
6217 if ((Subtarget->hasNEON())) {
6218 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
6219 }
6220 return Register();
6221}
6222
6223Register fastEmit_ISD_UADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6224 if (RetVT.SimpleTy != MVT::v2i64)
6225 return Register();
6226 if ((Subtarget->hasNEON())) {
6227 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
6228 }
6229 return Register();
6230}
6231
6232Register fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6233 switch (VT.SimpleTy) {
6234 case MVT::v8i8: return fastEmit_ISD_UADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
6235 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
6236 case MVT::v4i16: return fastEmit_ISD_UADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
6237 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
6238 case MVT::v2i32: return fastEmit_ISD_UADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
6239 case MVT::v4i32: return fastEmit_ISD_UADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
6240 case MVT::v1i64: return fastEmit_ISD_UADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
6241 case MVT::v2i64: return fastEmit_ISD_UADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
6242 default: return Register();
6243 }
6244}
6245
6246// FastEmit functions for ISD::UDIV.
6247
6248Register fastEmit_ISD_UDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6249 if (RetVT.SimpleTy != MVT::i32)
6250 return Register();
6251 if ((Subtarget->hasDivideInThumbMode()) && (Subtarget->hasV8MBaselineOps()) && (Subtarget->isThumb())) {
6252 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UDIV, RC: &ARM::rGPRRegClass, Op0, Op1);
6253 }
6254 if ((Subtarget->hasDivideInARMMode()) && (!Subtarget->isThumb())) {
6255 return fastEmitInst_rr(MachineInstOpcode: ARM::UDIV, RC: &ARM::GPRRegClass, Op0, Op1);
6256 }
6257 return Register();
6258}
6259
6260Register fastEmit_ISD_UDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6261 switch (VT.SimpleTy) {
6262 case MVT::i32: return fastEmit_ISD_UDIV_MVT_i32_rr(RetVT, Op0, Op1);
6263 default: return Register();
6264 }
6265}
6266
6267// FastEmit functions for ISD::UMAX.
6268
6269Register fastEmit_ISD_UMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6270 if (RetVT.SimpleTy != MVT::v8i8)
6271 return Register();
6272 if ((Subtarget->hasNEON())) {
6273 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6274 }
6275 return Register();
6276}
6277
6278Register fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6279 if (RetVT.SimpleTy != MVT::v16i8)
6280 return Register();
6281 if ((Subtarget->hasMVEIntegerOps())) {
6282 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXu8, RC: &ARM::MQPRRegClass, Op0, Op1);
6283 }
6284 if ((Subtarget->hasNEON())) {
6285 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6286 }
6287 return Register();
6288}
6289
6290Register fastEmit_ISD_UMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6291 if (RetVT.SimpleTy != MVT::v4i16)
6292 return Register();
6293 if ((Subtarget->hasNEON())) {
6294 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6295 }
6296 return Register();
6297}
6298
6299Register fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6300 if (RetVT.SimpleTy != MVT::v8i16)
6301 return Register();
6302 if ((Subtarget->hasMVEIntegerOps())) {
6303 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXu16, RC: &ARM::MQPRRegClass, Op0, Op1);
6304 }
6305 if ((Subtarget->hasNEON())) {
6306 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6307 }
6308 return Register();
6309}
6310
6311Register fastEmit_ISD_UMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6312 if (RetVT.SimpleTy != MVT::v2i32)
6313 return Register();
6314 if ((Subtarget->hasNEON())) {
6315 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6316 }
6317 return Register();
6318}
6319
6320Register fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6321 if (RetVT.SimpleTy != MVT::v4i32)
6322 return Register();
6323 if ((Subtarget->hasMVEIntegerOps())) {
6324 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXu32, RC: &ARM::MQPRRegClass, Op0, Op1);
6325 }
6326 if ((Subtarget->hasNEON())) {
6327 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6328 }
6329 return Register();
6330}
6331
6332Register fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6333 switch (VT.SimpleTy) {
6334 case MVT::v8i8: return fastEmit_ISD_UMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
6335 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
6336 case MVT::v4i16: return fastEmit_ISD_UMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
6337 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
6338 case MVT::v2i32: return fastEmit_ISD_UMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
6339 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
6340 default: return Register();
6341 }
6342}
6343
6344// FastEmit functions for ISD::UMIN.
6345
6346Register fastEmit_ISD_UMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6347 if (RetVT.SimpleTy != MVT::v8i8)
6348 return Register();
6349 if ((Subtarget->hasNEON())) {
6350 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6351 }
6352 return Register();
6353}
6354
6355Register fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6356 if (RetVT.SimpleTy != MVT::v16i8)
6357 return Register();
6358 if ((Subtarget->hasMVEIntegerOps())) {
6359 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINu8, RC: &ARM::MQPRRegClass, Op0, Op1);
6360 }
6361 if ((Subtarget->hasNEON())) {
6362 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6363 }
6364 return Register();
6365}
6366
6367Register fastEmit_ISD_UMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6368 if (RetVT.SimpleTy != MVT::v4i16)
6369 return Register();
6370 if ((Subtarget->hasNEON())) {
6371 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6372 }
6373 return Register();
6374}
6375
6376Register fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6377 if (RetVT.SimpleTy != MVT::v8i16)
6378 return Register();
6379 if ((Subtarget->hasMVEIntegerOps())) {
6380 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINu16, RC: &ARM::MQPRRegClass, Op0, Op1);
6381 }
6382 if ((Subtarget->hasNEON())) {
6383 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6384 }
6385 return Register();
6386}
6387
6388Register fastEmit_ISD_UMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6389 if (RetVT.SimpleTy != MVT::v2i32)
6390 return Register();
6391 if ((Subtarget->hasNEON())) {
6392 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6393 }
6394 return Register();
6395}
6396
6397Register fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6398 if (RetVT.SimpleTy != MVT::v4i32)
6399 return Register();
6400 if ((Subtarget->hasMVEIntegerOps())) {
6401 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINu32, RC: &ARM::MQPRRegClass, Op0, Op1);
6402 }
6403 if ((Subtarget->hasNEON())) {
6404 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6405 }
6406 return Register();
6407}
6408
6409Register fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6410 switch (VT.SimpleTy) {
6411 case MVT::v8i8: return fastEmit_ISD_UMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
6412 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
6413 case MVT::v4i16: return fastEmit_ISD_UMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
6414 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
6415 case MVT::v2i32: return fastEmit_ISD_UMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
6416 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
6417 default: return Register();
6418 }
6419}
6420
6421// FastEmit functions for ISD::USUBSAT.
6422
6423Register fastEmit_ISD_USUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6424 if (RetVT.SimpleTy != MVT::v8i8)
6425 return Register();
6426 if ((Subtarget->hasNEON())) {
6427 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6428 }
6429 return Register();
6430}
6431
6432Register fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6433 if (RetVT.SimpleTy != MVT::v16i8)
6434 return Register();
6435 if ((Subtarget->hasMVEIntegerOps())) {
6436 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBu8, RC: &ARM::MQPRRegClass, Op0, Op1);
6437 }
6438 if ((Subtarget->hasNEON())) {
6439 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6440 }
6441 return Register();
6442}
6443
6444Register fastEmit_ISD_USUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6445 if (RetVT.SimpleTy != MVT::v4i16)
6446 return Register();
6447 if ((Subtarget->hasNEON())) {
6448 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6449 }
6450 return Register();
6451}
6452
6453Register fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6454 if (RetVT.SimpleTy != MVT::v8i16)
6455 return Register();
6456 if ((Subtarget->hasMVEIntegerOps())) {
6457 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBu16, RC: &ARM::MQPRRegClass, Op0, Op1);
6458 }
6459 if ((Subtarget->hasNEON())) {
6460 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6461 }
6462 return Register();
6463}
6464
6465Register fastEmit_ISD_USUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6466 if (RetVT.SimpleTy != MVT::v2i32)
6467 return Register();
6468 if ((Subtarget->hasNEON())) {
6469 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6470 }
6471 return Register();
6472}
6473
6474Register fastEmit_ISD_USUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6475 if (RetVT.SimpleTy != MVT::v4i32)
6476 return Register();
6477 if ((Subtarget->hasMVEIntegerOps())) {
6478 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBu32, RC: &ARM::MQPRRegClass, Op0, Op1);
6479 }
6480 if ((Subtarget->hasNEON())) {
6481 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6482 }
6483 return Register();
6484}
6485
6486Register fastEmit_ISD_USUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
6487 if (RetVT.SimpleTy != MVT::v1i64)
6488 return Register();
6489 if ((Subtarget->hasNEON())) {
6490 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
6491 }
6492 return Register();
6493}
6494
6495Register fastEmit_ISD_USUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6496 if (RetVT.SimpleTy != MVT::v2i64)
6497 return Register();
6498 if ((Subtarget->hasNEON())) {
6499 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
6500 }
6501 return Register();
6502}
6503
6504Register fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6505 switch (VT.SimpleTy) {
6506 case MVT::v8i8: return fastEmit_ISD_USUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
6507 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
6508 case MVT::v4i16: return fastEmit_ISD_USUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
6509 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
6510 case MVT::v2i32: return fastEmit_ISD_USUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
6511 case MVT::v4i32: return fastEmit_ISD_USUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
6512 case MVT::v1i64: return fastEmit_ISD_USUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
6513 case MVT::v2i64: return fastEmit_ISD_USUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
6514 default: return Register();
6515 }
6516}
6517
6518// FastEmit functions for ISD::XOR.
6519
6520Register fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6521 if (RetVT.SimpleTy != MVT::i32)
6522 return Register();
6523 if ((Subtarget->isThumb2())) {
6524 return fastEmitInst_rr(MachineInstOpcode: ARM::t2EORrr, RC: &ARM::rGPRRegClass, Op0, Op1);
6525 }
6526 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6527 return fastEmitInst_rr(MachineInstOpcode: ARM::tEOR, RC: &ARM::tGPRRegClass, Op0, Op1);
6528 }
6529 if ((!Subtarget->isThumb())) {
6530 return fastEmitInst_rr(MachineInstOpcode: ARM::EORrr, RC: &ARM::GPRRegClass, Op0, Op1);
6531 }
6532 return Register();
6533}
6534
6535Register fastEmit_ISD_XOR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6536 if (RetVT.SimpleTy != MVT::v8i8)
6537 return Register();
6538 if ((Subtarget->hasNEON())) {
6539 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
6540 }
6541 return Register();
6542}
6543
6544Register fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6545 if (RetVT.SimpleTy != MVT::v16i8)
6546 return Register();
6547 if ((Subtarget->hasMVEIntegerOps())) {
6548 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
6549 }
6550 if ((Subtarget->hasNEON())) {
6551 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
6552 }
6553 return Register();
6554}
6555
6556Register fastEmit_ISD_XOR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6557 if (RetVT.SimpleTy != MVT::v4i16)
6558 return Register();
6559 if ((Subtarget->hasNEON())) {
6560 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
6561 }
6562 return Register();
6563}
6564
6565Register fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6566 if (RetVT.SimpleTy != MVT::v8i16)
6567 return Register();
6568 if ((Subtarget->hasMVEIntegerOps())) {
6569 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
6570 }
6571 if ((Subtarget->hasNEON())) {
6572 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
6573 }
6574 return Register();
6575}
6576
6577Register fastEmit_ISD_XOR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6578 if (RetVT.SimpleTy != MVT::v2i32)
6579 return Register();
6580 if ((Subtarget->hasNEON())) {
6581 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
6582 }
6583 return Register();
6584}
6585
6586Register fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6587 if (RetVT.SimpleTy != MVT::v4i32)
6588 return Register();
6589 if ((Subtarget->hasMVEIntegerOps())) {
6590 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
6591 }
6592 if ((Subtarget->hasNEON())) {
6593 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
6594 }
6595 return Register();
6596}
6597
6598Register fastEmit_ISD_XOR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
6599 if (RetVT.SimpleTy != MVT::v1i64)
6600 return Register();
6601 if ((Subtarget->hasNEON())) {
6602 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
6603 }
6604 return Register();
6605}
6606
6607Register fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6608 if (RetVT.SimpleTy != MVT::v2i64)
6609 return Register();
6610 if ((Subtarget->hasMVEIntegerOps())) {
6611 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
6612 }
6613 if ((Subtarget->hasNEON())) {
6614 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
6615 }
6616 return Register();
6617}
6618
6619Register fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6620 switch (VT.SimpleTy) {
6621 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
6622 case MVT::v8i8: return fastEmit_ISD_XOR_MVT_v8i8_rr(RetVT, Op0, Op1);
6623 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
6624 case MVT::v4i16: return fastEmit_ISD_XOR_MVT_v4i16_rr(RetVT, Op0, Op1);
6625 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
6626 case MVT::v2i32: return fastEmit_ISD_XOR_MVT_v2i32_rr(RetVT, Op0, Op1);
6627 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
6628 case MVT::v1i64: return fastEmit_ISD_XOR_MVT_v1i64_rr(RetVT, Op0, Op1);
6629 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
6630 default: return Register();
6631 }
6632}
6633
6634// Top-level FastEmit function.
6635
6636Register fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, Register Op1) override {
6637 switch (Opcode) {
6638 case ARMISD::CMP: return fastEmit_ARMISD_CMP_rr(VT, RetVT, Op0, Op1);
6639 case ARMISD::CMPFP: return fastEmit_ARMISD_CMPFP_rr(VT, RetVT, Op0, Op1);
6640 case ARMISD::CMPFPE: return fastEmit_ARMISD_CMPFPE_rr(VT, RetVT, Op0, Op1);
6641 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_rr(VT, RetVT, Op0, Op1);
6642 case ARMISD::EH_SJLJ_LONGJMP: return fastEmit_ARMISD_EH_SJLJ_LONGJMP_rr(VT, RetVT, Op0, Op1);
6643 case ARMISD::EH_SJLJ_SETJMP: return fastEmit_ARMISD_EH_SJLJ_SETJMP_rr(VT, RetVT, Op0, Op1);
6644 case ARMISD::QADD16b: return fastEmit_ARMISD_QADD16b_rr(VT, RetVT, Op0, Op1);
6645 case ARMISD::QADD8b: return fastEmit_ARMISD_QADD8b_rr(VT, RetVT, Op0, Op1);
6646 case ARMISD::QSUB16b: return fastEmit_ARMISD_QSUB16b_rr(VT, RetVT, Op0, Op1);
6647 case ARMISD::QSUB8b: return fastEmit_ARMISD_QSUB8b_rr(VT, RetVT, Op0, Op1);
6648 case ARMISD::SMULWB: return fastEmit_ARMISD_SMULWB_rr(VT, RetVT, Op0, Op1);
6649 case ARMISD::SMULWT: return fastEmit_ARMISD_SMULWT_rr(VT, RetVT, Op0, Op1);
6650 case ARMISD::UQADD16b: return fastEmit_ARMISD_UQADD16b_rr(VT, RetVT, Op0, Op1);
6651 case ARMISD::UQADD8b: return fastEmit_ARMISD_UQADD8b_rr(VT, RetVT, Op0, Op1);
6652 case ARMISD::UQSUB16b: return fastEmit_ARMISD_UQSUB16b_rr(VT, RetVT, Op0, Op1);
6653 case ARMISD::UQSUB8b: return fastEmit_ARMISD_UQSUB8b_rr(VT, RetVT, Op0, Op1);
6654 case ARMISD::VMLAVs: return fastEmit_ARMISD_VMLAVs_rr(VT, RetVT, Op0, Op1);
6655 case ARMISD::VMLAVu: return fastEmit_ARMISD_VMLAVu_rr(VT, RetVT, Op0, Op1);
6656 case ARMISD::VMOVDRR: return fastEmit_ARMISD_VMOVDRR_rr(VT, RetVT, Op0, Op1);
6657 case ARMISD::VMULLs: return fastEmit_ARMISD_VMULLs_rr(VT, RetVT, Op0, Op1);
6658 case ARMISD::VMULLu: return fastEmit_ARMISD_VMULLu_rr(VT, RetVT, Op0, Op1);
6659 case ARMISD::VQDMULH: return fastEmit_ARMISD_VQDMULH_rr(VT, RetVT, Op0, Op1);
6660 case ARMISD::VSHLs: return fastEmit_ARMISD_VSHLs_rr(VT, RetVT, Op0, Op1);
6661 case ARMISD::VSHLu: return fastEmit_ARMISD_VSHLu_rr(VT, RetVT, Op0, Op1);
6662 case ARMISD::VTBL1: return fastEmit_ARMISD_VTBL1_rr(VT, RetVT, Op0, Op1);
6663 case ARMISD::VTST: return fastEmit_ARMISD_VTST_rr(VT, RetVT, Op0, Op1);
6664 case ISD::ABDS: return fastEmit_ISD_ABDS_rr(VT, RetVT, Op0, Op1);
6665 case ISD::ABDU: return fastEmit_ISD_ABDU_rr(VT, RetVT, Op0, Op1);
6666 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
6667 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
6668 case ISD::AVGCEILS: return fastEmit_ISD_AVGCEILS_rr(VT, RetVT, Op0, Op1);
6669 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
6670 case ISD::AVGFLOORS: return fastEmit_ISD_AVGFLOORS_rr(VT, RetVT, Op0, Op1);
6671 case ISD::AVGFLOORU: return fastEmit_ISD_AVGFLOORU_rr(VT, RetVT, Op0, Op1);
6672 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
6673 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
6674 case ISD::FMAXIMUM: return fastEmit_ISD_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
6675 case ISD::FMAXNUM: return fastEmit_ISD_FMAXNUM_rr(VT, RetVT, Op0, Op1);
6676 case ISD::FMINIMUM: return fastEmit_ISD_FMINIMUM_rr(VT, RetVT, Op0, Op1);
6677 case ISD::FMINNUM: return fastEmit_ISD_FMINNUM_rr(VT, RetVT, Op0, Op1);
6678 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
6679 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
6680 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
6681 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
6682 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
6683 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
6684 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
6685 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
6686 case ISD::SDIV: return fastEmit_ISD_SDIV_rr(VT, RetVT, Op0, Op1);
6687 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
6688 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
6689 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
6690 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
6691 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
6692 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
6693 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
6694 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
6695 case ISD::UDIV: return fastEmit_ISD_UDIV_rr(VT, RetVT, Op0, Op1);
6696 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
6697 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
6698 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
6699 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
6700 default: return Register();
6701 }
6702}
6703
6704// FastEmit functions for ARMISD::PIC_ADD.
6705
6706Register fastEmit_ARMISD_PIC_ADD_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6707 if (RetVT.SimpleTy != MVT::i32)
6708 return Register();
6709 if ((Subtarget->isThumb())) {
6710 return fastEmitInst_ri(MachineInstOpcode: ARM::tPICADD, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
6711 }
6712 if ((!Subtarget->isThumb())) {
6713 return fastEmitInst_ri(MachineInstOpcode: ARM::PICADD, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
6714 }
6715 return Register();
6716}
6717
6718Register fastEmit_ARMISD_PIC_ADD_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
6719 switch (VT.SimpleTy) {
6720 case MVT::i32: return fastEmit_ARMISD_PIC_ADD_MVT_i32_ri(RetVT, Op0, imm1);
6721 default: return Register();
6722 }
6723}
6724
6725// FastEmit functions for ARMISD::VDUPLANE.
6726
6727Register fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6728 if (RetVT.SimpleTy != MVT::v8i8)
6729 return Register();
6730 if ((Subtarget->hasNEON())) {
6731 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN8d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6732 }
6733 return Register();
6734}
6735
6736Register fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6737 if (RetVT.SimpleTy != MVT::v4i16)
6738 return Register();
6739 if ((Subtarget->hasNEON())) {
6740 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6741 }
6742 return Register();
6743}
6744
6745Register fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6746 if (RetVT.SimpleTy != MVT::v2i32)
6747 return Register();
6748 if ((Subtarget->hasNEON())) {
6749 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6750 }
6751 return Register();
6752}
6753
6754Register fastEmit_ARMISD_VDUPLANE_MVT_v4f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6755 if (RetVT.SimpleTy != MVT::v4f16)
6756 return Register();
6757 if ((Subtarget->hasNEON())) {
6758 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6759 }
6760 return Register();
6761}
6762
6763Register fastEmit_ARMISD_VDUPLANE_MVT_v4bf16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6764 if (RetVT.SimpleTy != MVT::v4bf16)
6765 return Register();
6766 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
6767 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6768 }
6769 return Register();
6770}
6771
6772Register fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v2f32_ri(Register Op0, uint64_t imm1) {
6773 if ((Subtarget->hasNEON())) {
6774 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6775 }
6776 return Register();
6777}
6778
6779Register fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v4f32_ri(Register Op0, uint64_t imm1) {
6780 if ((Subtarget->hasNEON())) {
6781 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
6782 }
6783 return Register();
6784}
6785
6786Register fastEmit_ARMISD_VDUPLANE_MVT_v2f32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6787switch (RetVT.SimpleTy) {
6788 case MVT::v2f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v2f32_ri(Op0, imm1);
6789 case MVT::v4f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v4f32_ri(Op0, imm1);
6790 default: return Register();
6791}
6792}
6793
6794Register fastEmit_ARMISD_VDUPLANE_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
6795 switch (VT.SimpleTy) {
6796 case MVT::v8i8: return fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri(RetVT, Op0, imm1);
6797 case MVT::v4i16: return fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri(RetVT, Op0, imm1);
6798 case MVT::v2i32: return fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri(RetVT, Op0, imm1);
6799 case MVT::v4f16: return fastEmit_ARMISD_VDUPLANE_MVT_v4f16_ri(RetVT, Op0, imm1);
6800 case MVT::v4bf16: return fastEmit_ARMISD_VDUPLANE_MVT_v4bf16_ri(RetVT, Op0, imm1);
6801 case MVT::v2f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_ri(RetVT, Op0, imm1);
6802 default: return Register();
6803 }
6804}
6805
6806// FastEmit functions for ARMISD::VGETLANEs.
6807
6808Register fastEmit_ARMISD_VGETLANEs_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6809 if (RetVT.SimpleTy != MVT::i32)
6810 return Register();
6811 if ((Subtarget->hasNEON())) {
6812 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNs8, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
6813 }
6814 return Register();
6815}
6816
6817Register fastEmit_ARMISD_VGETLANEs_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6818 if (RetVT.SimpleTy != MVT::i32)
6819 return Register();
6820 if ((Subtarget->hasMVEIntegerOps())) {
6821 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_s8, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
6822 }
6823 return Register();
6824}
6825
6826Register fastEmit_ARMISD_VGETLANEs_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6827 if (RetVT.SimpleTy != MVT::i32)
6828 return Register();
6829 if ((Subtarget->hasNEON())) {
6830 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNs16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
6831 }
6832 return Register();
6833}
6834
6835Register fastEmit_ARMISD_VGETLANEs_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6836 if (RetVT.SimpleTy != MVT::i32)
6837 return Register();
6838 if ((Subtarget->hasMVEIntegerOps())) {
6839 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_s16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
6840 }
6841 return Register();
6842}
6843
6844Register fastEmit_ARMISD_VGETLANEs_MVT_v8f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6845 if (RetVT.SimpleTy != MVT::i32)
6846 return Register();
6847 if ((Subtarget->hasMVEIntegerOps())) {
6848 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_s16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
6849 }
6850 return Register();
6851}
6852
6853Register fastEmit_ARMISD_VGETLANEs_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
6854 switch (VT.SimpleTy) {
6855 case MVT::v8i8: return fastEmit_ARMISD_VGETLANEs_MVT_v8i8_ri(RetVT, Op0, imm1);
6856 case MVT::v16i8: return fastEmit_ARMISD_VGETLANEs_MVT_v16i8_ri(RetVT, Op0, imm1);
6857 case MVT::v4i16: return fastEmit_ARMISD_VGETLANEs_MVT_v4i16_ri(RetVT, Op0, imm1);
6858 case MVT::v8i16: return fastEmit_ARMISD_VGETLANEs_MVT_v8i16_ri(RetVT, Op0, imm1);
6859 case MVT::v8f16: return fastEmit_ARMISD_VGETLANEs_MVT_v8f16_ri(RetVT, Op0, imm1);
6860 default: return Register();
6861 }
6862}
6863
6864// FastEmit functions for ARMISD::VGETLANEu.
6865
6866Register fastEmit_ARMISD_VGETLANEu_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6867 if (RetVT.SimpleTy != MVT::i32)
6868 return Register();
6869 if ((Subtarget->hasNEON())) {
6870 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu8, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
6871 }
6872 return Register();
6873}
6874
6875Register fastEmit_ARMISD_VGETLANEu_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6876 if (RetVT.SimpleTy != MVT::i32)
6877 return Register();
6878 if ((Subtarget->hasMVEIntegerOps())) {
6879 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_u8, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
6880 }
6881 return Register();
6882}
6883
6884Register fastEmit_ARMISD_VGETLANEu_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6885 if (RetVT.SimpleTy != MVT::i32)
6886 return Register();
6887 if ((Subtarget->hasNEON())) {
6888 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
6889 }
6890 return Register();
6891}
6892
6893Register fastEmit_ARMISD_VGETLANEu_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6894 if (RetVT.SimpleTy != MVT::i32)
6895 return Register();
6896 if ((Subtarget->hasMVEIntegerOps())) {
6897 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_u16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
6898 }
6899 return Register();
6900}
6901
6902Register fastEmit_ARMISD_VGETLANEu_MVT_v4f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6903 if (RetVT.SimpleTy != MVT::i32)
6904 return Register();
6905 if ((Subtarget->hasNEON())) {
6906 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
6907 }
6908 return Register();
6909}
6910
6911Register fastEmit_ARMISD_VGETLANEu_MVT_v8f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6912 if (RetVT.SimpleTy != MVT::i32)
6913 return Register();
6914 if ((Subtarget->hasMVEIntegerOps())) {
6915 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_u16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
6916 }
6917 return Register();
6918}
6919
6920Register fastEmit_ARMISD_VGETLANEu_MVT_v4bf16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6921 if (RetVT.SimpleTy != MVT::i32)
6922 return Register();
6923 if ((Subtarget->hasNEON())) {
6924 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
6925 }
6926 return Register();
6927}
6928
6929Register fastEmit_ARMISD_VGETLANEu_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
6930 switch (VT.SimpleTy) {
6931 case MVT::v8i8: return fastEmit_ARMISD_VGETLANEu_MVT_v8i8_ri(RetVT, Op0, imm1);
6932 case MVT::v16i8: return fastEmit_ARMISD_VGETLANEu_MVT_v16i8_ri(RetVT, Op0, imm1);
6933 case MVT::v4i16: return fastEmit_ARMISD_VGETLANEu_MVT_v4i16_ri(RetVT, Op0, imm1);
6934 case MVT::v8i16: return fastEmit_ARMISD_VGETLANEu_MVT_v8i16_ri(RetVT, Op0, imm1);
6935 case MVT::v4f16: return fastEmit_ARMISD_VGETLANEu_MVT_v4f16_ri(RetVT, Op0, imm1);
6936 case MVT::v8f16: return fastEmit_ARMISD_VGETLANEu_MVT_v8f16_ri(RetVT, Op0, imm1);
6937 case MVT::v4bf16: return fastEmit_ARMISD_VGETLANEu_MVT_v4bf16_ri(RetVT, Op0, imm1);
6938 default: return Register();
6939 }
6940}
6941
6942// FastEmit functions for ARMISD::VQSHLsIMM.
6943
6944Register fastEmit_ARMISD_VQSHLsIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6945 if (RetVT.SimpleTy != MVT::v8i8)
6946 return Register();
6947 if ((Subtarget->hasNEON())) {
6948 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6949 }
6950 return Register();
6951}
6952
6953Register fastEmit_ARMISD_VQSHLsIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6954 if (RetVT.SimpleTy != MVT::v16i8)
6955 return Register();
6956 if ((Subtarget->hasNEON())) {
6957 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
6958 }
6959 return Register();
6960}
6961
6962Register fastEmit_ARMISD_VQSHLsIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6963 if (RetVT.SimpleTy != MVT::v4i16)
6964 return Register();
6965 if ((Subtarget->hasNEON())) {
6966 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6967 }
6968 return Register();
6969}
6970
6971Register fastEmit_ARMISD_VQSHLsIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6972 if (RetVT.SimpleTy != MVT::v8i16)
6973 return Register();
6974 if ((Subtarget->hasNEON())) {
6975 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
6976 }
6977 return Register();
6978}
6979
6980Register fastEmit_ARMISD_VQSHLsIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6981 if (RetVT.SimpleTy != MVT::v2i32)
6982 return Register();
6983 if ((Subtarget->hasNEON())) {
6984 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6985 }
6986 return Register();
6987}
6988
6989Register fastEmit_ARMISD_VQSHLsIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6990 if (RetVT.SimpleTy != MVT::v4i32)
6991 return Register();
6992 if ((Subtarget->hasNEON())) {
6993 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
6994 }
6995 return Register();
6996}
6997
6998Register fastEmit_ARMISD_VQSHLsIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
6999 if (RetVT.SimpleTy != MVT::v1i64)
7000 return Register();
7001 if ((Subtarget->hasNEON())) {
7002 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7003 }
7004 return Register();
7005}
7006
7007Register fastEmit_ARMISD_VQSHLsIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7008 if (RetVT.SimpleTy != MVT::v2i64)
7009 return Register();
7010 if ((Subtarget->hasNEON())) {
7011 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7012 }
7013 return Register();
7014}
7015
7016Register fastEmit_ARMISD_VQSHLsIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7017 switch (VT.SimpleTy) {
7018 case MVT::v8i8: return fastEmit_ARMISD_VQSHLsIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7019 case MVT::v16i8: return fastEmit_ARMISD_VQSHLsIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7020 case MVT::v4i16: return fastEmit_ARMISD_VQSHLsIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7021 case MVT::v8i16: return fastEmit_ARMISD_VQSHLsIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7022 case MVT::v2i32: return fastEmit_ARMISD_VQSHLsIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7023 case MVT::v4i32: return fastEmit_ARMISD_VQSHLsIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7024 case MVT::v1i64: return fastEmit_ARMISD_VQSHLsIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7025 case MVT::v2i64: return fastEmit_ARMISD_VQSHLsIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7026 default: return Register();
7027 }
7028}
7029
7030// FastEmit functions for ARMISD::VQSHLsuIMM.
7031
7032Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7033 if (RetVT.SimpleTy != MVT::v8i8)
7034 return Register();
7035 if ((Subtarget->hasNEON())) {
7036 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7037 }
7038 return Register();
7039}
7040
7041Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7042 if (RetVT.SimpleTy != MVT::v16i8)
7043 return Register();
7044 if ((Subtarget->hasNEON())) {
7045 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7046 }
7047 return Register();
7048}
7049
7050Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7051 if (RetVT.SimpleTy != MVT::v4i16)
7052 return Register();
7053 if ((Subtarget->hasNEON())) {
7054 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7055 }
7056 return Register();
7057}
7058
7059Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7060 if (RetVT.SimpleTy != MVT::v8i16)
7061 return Register();
7062 if ((Subtarget->hasNEON())) {
7063 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7064 }
7065 return Register();
7066}
7067
7068Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7069 if (RetVT.SimpleTy != MVT::v2i32)
7070 return Register();
7071 if ((Subtarget->hasNEON())) {
7072 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7073 }
7074 return Register();
7075}
7076
7077Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7078 if (RetVT.SimpleTy != MVT::v4i32)
7079 return Register();
7080 if ((Subtarget->hasNEON())) {
7081 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7082 }
7083 return Register();
7084}
7085
7086Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7087 if (RetVT.SimpleTy != MVT::v1i64)
7088 return Register();
7089 if ((Subtarget->hasNEON())) {
7090 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7091 }
7092 return Register();
7093}
7094
7095Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7096 if (RetVT.SimpleTy != MVT::v2i64)
7097 return Register();
7098 if ((Subtarget->hasNEON())) {
7099 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7100 }
7101 return Register();
7102}
7103
7104Register fastEmit_ARMISD_VQSHLsuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7105 switch (VT.SimpleTy) {
7106 case MVT::v8i8: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7107 case MVT::v16i8: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7108 case MVT::v4i16: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7109 case MVT::v8i16: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7110 case MVT::v2i32: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7111 case MVT::v4i32: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7112 case MVT::v1i64: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7113 case MVT::v2i64: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7114 default: return Register();
7115 }
7116}
7117
7118// FastEmit functions for ARMISD::VQSHLuIMM.
7119
7120Register fastEmit_ARMISD_VQSHLuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7121 if (RetVT.SimpleTy != MVT::v8i8)
7122 return Register();
7123 if ((Subtarget->hasNEON())) {
7124 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7125 }
7126 return Register();
7127}
7128
7129Register fastEmit_ARMISD_VQSHLuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7130 if (RetVT.SimpleTy != MVT::v16i8)
7131 return Register();
7132 if ((Subtarget->hasNEON())) {
7133 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7134 }
7135 return Register();
7136}
7137
7138Register fastEmit_ARMISD_VQSHLuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7139 if (RetVT.SimpleTy != MVT::v4i16)
7140 return Register();
7141 if ((Subtarget->hasNEON())) {
7142 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7143 }
7144 return Register();
7145}
7146
7147Register fastEmit_ARMISD_VQSHLuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7148 if (RetVT.SimpleTy != MVT::v8i16)
7149 return Register();
7150 if ((Subtarget->hasNEON())) {
7151 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7152 }
7153 return Register();
7154}
7155
7156Register fastEmit_ARMISD_VQSHLuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7157 if (RetVT.SimpleTy != MVT::v2i32)
7158 return Register();
7159 if ((Subtarget->hasNEON())) {
7160 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7161 }
7162 return Register();
7163}
7164
7165Register fastEmit_ARMISD_VQSHLuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7166 if (RetVT.SimpleTy != MVT::v4i32)
7167 return Register();
7168 if ((Subtarget->hasNEON())) {
7169 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7170 }
7171 return Register();
7172}
7173
7174Register fastEmit_ARMISD_VQSHLuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7175 if (RetVT.SimpleTy != MVT::v1i64)
7176 return Register();
7177 if ((Subtarget->hasNEON())) {
7178 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7179 }
7180 return Register();
7181}
7182
7183Register fastEmit_ARMISD_VQSHLuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7184 if (RetVT.SimpleTy != MVT::v2i64)
7185 return Register();
7186 if ((Subtarget->hasNEON())) {
7187 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7188 }
7189 return Register();
7190}
7191
7192Register fastEmit_ARMISD_VQSHLuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7193 switch (VT.SimpleTy) {
7194 case MVT::v8i8: return fastEmit_ARMISD_VQSHLuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7195 case MVT::v16i8: return fastEmit_ARMISD_VQSHLuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7196 case MVT::v4i16: return fastEmit_ARMISD_VQSHLuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7197 case MVT::v8i16: return fastEmit_ARMISD_VQSHLuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7198 case MVT::v2i32: return fastEmit_ARMISD_VQSHLuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7199 case MVT::v4i32: return fastEmit_ARMISD_VQSHLuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7200 case MVT::v1i64: return fastEmit_ARMISD_VQSHLuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7201 case MVT::v2i64: return fastEmit_ARMISD_VQSHLuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7202 default: return Register();
7203 }
7204}
7205
7206// FastEmit functions for ARMISD::VRSHRsIMM.
7207
7208Register fastEmit_ARMISD_VRSHRsIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7209 if (RetVT.SimpleTy != MVT::v8i8)
7210 return Register();
7211 if ((Subtarget->hasNEON())) {
7212 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7213 }
7214 return Register();
7215}
7216
7217Register fastEmit_ARMISD_VRSHRsIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7218 if (RetVT.SimpleTy != MVT::v16i8)
7219 return Register();
7220 if ((Subtarget->hasNEON())) {
7221 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7222 }
7223 return Register();
7224}
7225
7226Register fastEmit_ARMISD_VRSHRsIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7227 if (RetVT.SimpleTy != MVT::v4i16)
7228 return Register();
7229 if ((Subtarget->hasNEON())) {
7230 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7231 }
7232 return Register();
7233}
7234
7235Register fastEmit_ARMISD_VRSHRsIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7236 if (RetVT.SimpleTy != MVT::v8i16)
7237 return Register();
7238 if ((Subtarget->hasNEON())) {
7239 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7240 }
7241 return Register();
7242}
7243
7244Register fastEmit_ARMISD_VRSHRsIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7245 if (RetVT.SimpleTy != MVT::v2i32)
7246 return Register();
7247 if ((Subtarget->hasNEON())) {
7248 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7249 }
7250 return Register();
7251}
7252
7253Register fastEmit_ARMISD_VRSHRsIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7254 if (RetVT.SimpleTy != MVT::v4i32)
7255 return Register();
7256 if ((Subtarget->hasNEON())) {
7257 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7258 }
7259 return Register();
7260}
7261
7262Register fastEmit_ARMISD_VRSHRsIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7263 if (RetVT.SimpleTy != MVT::v1i64)
7264 return Register();
7265 if ((Subtarget->hasNEON())) {
7266 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7267 }
7268 return Register();
7269}
7270
7271Register fastEmit_ARMISD_VRSHRsIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7272 if (RetVT.SimpleTy != MVT::v2i64)
7273 return Register();
7274 if ((Subtarget->hasNEON())) {
7275 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7276 }
7277 return Register();
7278}
7279
7280Register fastEmit_ARMISD_VRSHRsIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7281 switch (VT.SimpleTy) {
7282 case MVT::v8i8: return fastEmit_ARMISD_VRSHRsIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7283 case MVT::v16i8: return fastEmit_ARMISD_VRSHRsIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7284 case MVT::v4i16: return fastEmit_ARMISD_VRSHRsIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7285 case MVT::v8i16: return fastEmit_ARMISD_VRSHRsIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7286 case MVT::v2i32: return fastEmit_ARMISD_VRSHRsIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7287 case MVT::v4i32: return fastEmit_ARMISD_VRSHRsIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7288 case MVT::v1i64: return fastEmit_ARMISD_VRSHRsIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7289 case MVT::v2i64: return fastEmit_ARMISD_VRSHRsIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7290 default: return Register();
7291 }
7292}
7293
7294// FastEmit functions for ARMISD::VRSHRuIMM.
7295
7296Register fastEmit_ARMISD_VRSHRuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7297 if (RetVT.SimpleTy != MVT::v8i8)
7298 return Register();
7299 if ((Subtarget->hasNEON())) {
7300 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7301 }
7302 return Register();
7303}
7304
7305Register fastEmit_ARMISD_VRSHRuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7306 if (RetVT.SimpleTy != MVT::v16i8)
7307 return Register();
7308 if ((Subtarget->hasNEON())) {
7309 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7310 }
7311 return Register();
7312}
7313
7314Register fastEmit_ARMISD_VRSHRuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7315 if (RetVT.SimpleTy != MVT::v4i16)
7316 return Register();
7317 if ((Subtarget->hasNEON())) {
7318 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7319 }
7320 return Register();
7321}
7322
7323Register fastEmit_ARMISD_VRSHRuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7324 if (RetVT.SimpleTy != MVT::v8i16)
7325 return Register();
7326 if ((Subtarget->hasNEON())) {
7327 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7328 }
7329 return Register();
7330}
7331
7332Register fastEmit_ARMISD_VRSHRuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7333 if (RetVT.SimpleTy != MVT::v2i32)
7334 return Register();
7335 if ((Subtarget->hasNEON())) {
7336 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7337 }
7338 return Register();
7339}
7340
7341Register fastEmit_ARMISD_VRSHRuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7342 if (RetVT.SimpleTy != MVT::v4i32)
7343 return Register();
7344 if ((Subtarget->hasNEON())) {
7345 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7346 }
7347 return Register();
7348}
7349
7350Register fastEmit_ARMISD_VRSHRuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7351 if (RetVT.SimpleTy != MVT::v1i64)
7352 return Register();
7353 if ((Subtarget->hasNEON())) {
7354 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7355 }
7356 return Register();
7357}
7358
7359Register fastEmit_ARMISD_VRSHRuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7360 if (RetVT.SimpleTy != MVT::v2i64)
7361 return Register();
7362 if ((Subtarget->hasNEON())) {
7363 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7364 }
7365 return Register();
7366}
7367
7368Register fastEmit_ARMISD_VRSHRuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7369 switch (VT.SimpleTy) {
7370 case MVT::v8i8: return fastEmit_ARMISD_VRSHRuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7371 case MVT::v16i8: return fastEmit_ARMISD_VRSHRuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7372 case MVT::v4i16: return fastEmit_ARMISD_VRSHRuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7373 case MVT::v8i16: return fastEmit_ARMISD_VRSHRuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7374 case MVT::v2i32: return fastEmit_ARMISD_VRSHRuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7375 case MVT::v4i32: return fastEmit_ARMISD_VRSHRuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7376 case MVT::v1i64: return fastEmit_ARMISD_VRSHRuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7377 case MVT::v2i64: return fastEmit_ARMISD_VRSHRuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7378 default: return Register();
7379 }
7380}
7381
7382// FastEmit functions for ARMISD::VSHLIMM.
7383
7384Register fastEmit_ARMISD_VSHLIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7385 if (RetVT.SimpleTy != MVT::v8i8)
7386 return Register();
7387 if ((Subtarget->hasNEON())) {
7388 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7389 }
7390 return Register();
7391}
7392
7393Register fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7394 if (RetVT.SimpleTy != MVT::v16i8)
7395 return Register();
7396 if ((Subtarget->hasNEON())) {
7397 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7398 }
7399 return Register();
7400}
7401
7402Register fastEmit_ARMISD_VSHLIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7403 if (RetVT.SimpleTy != MVT::v4i16)
7404 return Register();
7405 if ((Subtarget->hasNEON())) {
7406 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7407 }
7408 return Register();
7409}
7410
7411Register fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7412 if (RetVT.SimpleTy != MVT::v8i16)
7413 return Register();
7414 if ((Subtarget->hasNEON())) {
7415 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7416 }
7417 return Register();
7418}
7419
7420Register fastEmit_ARMISD_VSHLIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7421 if (RetVT.SimpleTy != MVT::v2i32)
7422 return Register();
7423 if ((Subtarget->hasNEON())) {
7424 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7425 }
7426 return Register();
7427}
7428
7429Register fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7430 if (RetVT.SimpleTy != MVT::v4i32)
7431 return Register();
7432 if ((Subtarget->hasNEON())) {
7433 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7434 }
7435 return Register();
7436}
7437
7438Register fastEmit_ARMISD_VSHLIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7439 if (RetVT.SimpleTy != MVT::v1i64)
7440 return Register();
7441 if ((Subtarget->hasNEON())) {
7442 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7443 }
7444 return Register();
7445}
7446
7447Register fastEmit_ARMISD_VSHLIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7448 if (RetVT.SimpleTy != MVT::v2i64)
7449 return Register();
7450 if ((Subtarget->hasNEON())) {
7451 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7452 }
7453 return Register();
7454}
7455
7456Register fastEmit_ARMISD_VSHLIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7457 switch (VT.SimpleTy) {
7458 case MVT::v8i8: return fastEmit_ARMISD_VSHLIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7459 case MVT::v16i8: return fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7460 case MVT::v4i16: return fastEmit_ARMISD_VSHLIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7461 case MVT::v8i16: return fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7462 case MVT::v2i32: return fastEmit_ARMISD_VSHLIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7463 case MVT::v4i32: return fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7464 case MVT::v1i64: return fastEmit_ARMISD_VSHLIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7465 case MVT::v2i64: return fastEmit_ARMISD_VSHLIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7466 default: return Register();
7467 }
7468}
7469
7470// FastEmit functions for ARMISD::VSHRsIMM.
7471
7472Register fastEmit_ARMISD_VSHRsIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7473 if (RetVT.SimpleTy != MVT::v8i8)
7474 return Register();
7475 if ((Subtarget->hasNEON())) {
7476 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7477 }
7478 return Register();
7479}
7480
7481Register fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7482 if (RetVT.SimpleTy != MVT::v16i8)
7483 return Register();
7484 if ((Subtarget->hasNEON())) {
7485 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7486 }
7487 return Register();
7488}
7489
7490Register fastEmit_ARMISD_VSHRsIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7491 if (RetVT.SimpleTy != MVT::v4i16)
7492 return Register();
7493 if ((Subtarget->hasNEON())) {
7494 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7495 }
7496 return Register();
7497}
7498
7499Register fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7500 if (RetVT.SimpleTy != MVT::v8i16)
7501 return Register();
7502 if ((Subtarget->hasNEON())) {
7503 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7504 }
7505 return Register();
7506}
7507
7508Register fastEmit_ARMISD_VSHRsIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7509 if (RetVT.SimpleTy != MVT::v2i32)
7510 return Register();
7511 if ((Subtarget->hasNEON())) {
7512 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7513 }
7514 return Register();
7515}
7516
7517Register fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7518 if (RetVT.SimpleTy != MVT::v4i32)
7519 return Register();
7520 if ((Subtarget->hasNEON())) {
7521 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7522 }
7523 return Register();
7524}
7525
7526Register fastEmit_ARMISD_VSHRsIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7527 if (RetVT.SimpleTy != MVT::v1i64)
7528 return Register();
7529 if ((Subtarget->hasNEON())) {
7530 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7531 }
7532 return Register();
7533}
7534
7535Register fastEmit_ARMISD_VSHRsIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7536 if (RetVT.SimpleTy != MVT::v2i64)
7537 return Register();
7538 if ((Subtarget->hasNEON())) {
7539 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7540 }
7541 return Register();
7542}
7543
7544Register fastEmit_ARMISD_VSHRsIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7545 switch (VT.SimpleTy) {
7546 case MVT::v8i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7547 case MVT::v16i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7548 case MVT::v4i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7549 case MVT::v8i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7550 case MVT::v2i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7551 case MVT::v4i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7552 case MVT::v1i64: return fastEmit_ARMISD_VSHRsIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7553 case MVT::v2i64: return fastEmit_ARMISD_VSHRsIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7554 default: return Register();
7555 }
7556}
7557
7558// FastEmit functions for ARMISD::VSHRuIMM.
7559
7560Register fastEmit_ARMISD_VSHRuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7561 if (RetVT.SimpleTy != MVT::v8i8)
7562 return Register();
7563 if ((Subtarget->hasNEON())) {
7564 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7565 }
7566 return Register();
7567}
7568
7569Register fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7570 if (RetVT.SimpleTy != MVT::v16i8)
7571 return Register();
7572 if ((Subtarget->hasNEON())) {
7573 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7574 }
7575 return Register();
7576}
7577
7578Register fastEmit_ARMISD_VSHRuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7579 if (RetVT.SimpleTy != MVT::v4i16)
7580 return Register();
7581 if ((Subtarget->hasNEON())) {
7582 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7583 }
7584 return Register();
7585}
7586
7587Register fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7588 if (RetVT.SimpleTy != MVT::v8i16)
7589 return Register();
7590 if ((Subtarget->hasNEON())) {
7591 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7592 }
7593 return Register();
7594}
7595
7596Register fastEmit_ARMISD_VSHRuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7597 if (RetVT.SimpleTy != MVT::v2i32)
7598 return Register();
7599 if ((Subtarget->hasNEON())) {
7600 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7601 }
7602 return Register();
7603}
7604
7605Register fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7606 if (RetVT.SimpleTy != MVT::v4i32)
7607 return Register();
7608 if ((Subtarget->hasNEON())) {
7609 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7610 }
7611 return Register();
7612}
7613
7614Register fastEmit_ARMISD_VSHRuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7615 if (RetVT.SimpleTy != MVT::v1i64)
7616 return Register();
7617 if ((Subtarget->hasNEON())) {
7618 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7619 }
7620 return Register();
7621}
7622
7623Register fastEmit_ARMISD_VSHRuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7624 if (RetVT.SimpleTy != MVT::v2i64)
7625 return Register();
7626 if ((Subtarget->hasNEON())) {
7627 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7628 }
7629 return Register();
7630}
7631
7632Register fastEmit_ARMISD_VSHRuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7633 switch (VT.SimpleTy) {
7634 case MVT::v8i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7635 case MVT::v16i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7636 case MVT::v4i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7637 case MVT::v8i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7638 case MVT::v2i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7639 case MVT::v4i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7640 case MVT::v1i64: return fastEmit_ARMISD_VSHRuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7641 case MVT::v2i64: return fastEmit_ARMISD_VSHRuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7642 default: return Register();
7643 }
7644}
7645
7646// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
7647
7648Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7649 if (RetVT.SimpleTy != MVT::i32)
7650 return Register();
7651 if ((Subtarget->hasFPRegs()) && (!Subtarget->hasSlowVGETLNi32())) {
7652 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNi32, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7653 }
7654 return Register();
7655}
7656
7657Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7658 switch (VT.SimpleTy) {
7659 case MVT::v2i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i32_ri(RetVT, Op0, imm1);
7660 default: return Register();
7661 }
7662}
7663
7664// FastEmit functions for ISD::SHL.
7665
7666Register fastEmit_ISD_SHL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7667 if (RetVT.SimpleTy != MVT::i32)
7668 return Register();
7669 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
7670 return fastEmitInst_ri(MachineInstOpcode: ARM::tLSLri, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
7671 }
7672 return Register();
7673}
7674
7675Register fastEmit_ISD_SHL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7676 switch (VT.SimpleTy) {
7677 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri(RetVT, Op0, imm1);
7678 default: return Register();
7679 }
7680}
7681
7682// Top-level FastEmit function.
7683
7684Register fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) override {
7685 if (VT == MVT::i32 && Predicate_mod_imm(Imm: imm1))
7686 if (Register Reg = fastEmit_ri_Predicate_mod_imm(VT, RetVT, Opcode, Op0, imm1))
7687 return Reg;
7688
7689 if (VT == MVT::i32 && Predicate_imm0_7(Imm: imm1))
7690 if (Register Reg = fastEmit_ri_Predicate_imm0_7(VT, RetVT, Opcode, Op0, imm1))
7691 return Reg;
7692
7693 if (VT == MVT::i32 && Predicate_imm0_255_expr(Imm: imm1))
7694 if (Register Reg = fastEmit_ri_Predicate_imm0_255_expr(VT, RetVT, Opcode, Op0, imm1))
7695 return Reg;
7696
7697 if (VT == MVT::i32 && Predicate_imm0_255(Imm: imm1))
7698 if (Register Reg = fastEmit_ri_Predicate_imm0_255(VT, RetVT, Opcode, Op0, imm1))
7699 return Reg;
7700
7701 if (VT == MVT::i32 && Predicate_t2_so_imm(Imm: imm1))
7702 if (Register Reg = fastEmit_ri_Predicate_t2_so_imm(VT, RetVT, Opcode, Op0, imm1))
7703 return Reg;
7704
7705 if (VT == MVT::i32 && Predicate_imm0_4095(Imm: imm1))
7706 if (Register Reg = fastEmit_ri_Predicate_imm0_4095(VT, RetVT, Opcode, Op0, imm1))
7707 return Reg;
7708
7709 if (VT == MVT::i32 && Predicate_imm1_31(Imm: imm1))
7710 if (Register Reg = fastEmit_ri_Predicate_imm1_31(VT, RetVT, Opcode, Op0, imm1))
7711 return Reg;
7712
7713 if (VT == MVT::i32 && Predicate_shr_imm8(Imm: imm1))
7714 if (Register Reg = fastEmit_ri_Predicate_shr_imm8(VT, RetVT, Opcode, Op0, imm1))
7715 return Reg;
7716
7717 if (VT == MVT::i32 && Predicate_shr_imm16(Imm: imm1))
7718 if (Register Reg = fastEmit_ri_Predicate_shr_imm16(VT, RetVT, Opcode, Op0, imm1))
7719 return Reg;
7720
7721 if (VT == MVT::i32 && Predicate_shr_imm32(Imm: imm1))
7722 if (Register Reg = fastEmit_ri_Predicate_shr_imm32(VT, RetVT, Opcode, Op0, imm1))
7723 return Reg;
7724
7725 if (VT == MVT::i32 && Predicate_VectorIndex32(Imm: imm1))
7726 if (Register Reg = fastEmit_ri_Predicate_VectorIndex32(VT, RetVT, Opcode, Op0, imm1))
7727 return Reg;
7728
7729 if (VT == MVT::i32 && Predicate_imm0_31(Imm: imm1))
7730 if (Register Reg = fastEmit_ri_Predicate_imm0_31(VT, RetVT, Opcode, Op0, imm1))
7731 return Reg;
7732
7733 if (VT == MVT::i32 && Predicate_imm0_15(Imm: imm1))
7734 if (Register Reg = fastEmit_ri_Predicate_imm0_15(VT, RetVT, Opcode, Op0, imm1))
7735 return Reg;
7736
7737 switch (Opcode) {
7738 case ARMISD::PIC_ADD: return fastEmit_ARMISD_PIC_ADD_ri(VT, RetVT, Op0, imm1);
7739 case ARMISD::VDUPLANE: return fastEmit_ARMISD_VDUPLANE_ri(VT, RetVT, Op0, imm1);
7740 case ARMISD::VGETLANEs: return fastEmit_ARMISD_VGETLANEs_ri(VT, RetVT, Op0, imm1);
7741 case ARMISD::VGETLANEu: return fastEmit_ARMISD_VGETLANEu_ri(VT, RetVT, Op0, imm1);
7742 case ARMISD::VQSHLsIMM: return fastEmit_ARMISD_VQSHLsIMM_ri(VT, RetVT, Op0, imm1);
7743 case ARMISD::VQSHLsuIMM: return fastEmit_ARMISD_VQSHLsuIMM_ri(VT, RetVT, Op0, imm1);
7744 case ARMISD::VQSHLuIMM: return fastEmit_ARMISD_VQSHLuIMM_ri(VT, RetVT, Op0, imm1);
7745 case ARMISD::VRSHRsIMM: return fastEmit_ARMISD_VRSHRsIMM_ri(VT, RetVT, Op0, imm1);
7746 case ARMISD::VRSHRuIMM: return fastEmit_ARMISD_VRSHRuIMM_ri(VT, RetVT, Op0, imm1);
7747 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri(VT, RetVT, Op0, imm1);
7748 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri(VT, RetVT, Op0, imm1);
7749 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri(VT, RetVT, Op0, imm1);
7750 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(VT, RetVT, Op0, imm1);
7751 case ISD::SHL: return fastEmit_ISD_SHL_ri(VT, RetVT, Op0, imm1);
7752 default: return Register();
7753 }
7754}
7755
7756// FastEmit functions for ARMISD::CMN.
7757
7758Register fastEmit_ARMISD_CMN_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
7759 if (RetVT.SimpleTy != MVT::i32)
7760 return Register();
7761 if ((!Subtarget->isThumb())) {
7762 return fastEmitInst_ri(MachineInstOpcode: ARM::CMNri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7763 }
7764 return Register();
7765}
7766
7767Register fastEmit_ARMISD_CMN_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7768 switch (VT.SimpleTy) {
7769 case MVT::i32: return fastEmit_ARMISD_CMN_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
7770 default: return Register();
7771 }
7772}
7773
7774// FastEmit functions for ARMISD::CMP.
7775
7776Register fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
7777 if (RetVT.SimpleTy != MVT::i32)
7778 return Register();
7779 if ((!Subtarget->isThumb())) {
7780 return fastEmitInst_ri(MachineInstOpcode: ARM::CMPri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7781 }
7782 return Register();
7783}
7784
7785Register fastEmit_ARMISD_CMP_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7786 switch (VT.SimpleTy) {
7787 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
7788 default: return Register();
7789 }
7790}
7791
7792// FastEmit functions for ARMISD::CMPZ.
7793
7794Register fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
7795 if (RetVT.SimpleTy != MVT::i32)
7796 return Register();
7797 if ((!Subtarget->isThumb())) {
7798 return fastEmitInst_ri(MachineInstOpcode: ARM::CMPri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7799 }
7800 return Register();
7801}
7802
7803Register fastEmit_ARMISD_CMPZ_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7804 switch (VT.SimpleTy) {
7805 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
7806 default: return Register();
7807 }
7808}
7809
7810// FastEmit functions for ISD::ADD.
7811
7812Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
7813 if (RetVT.SimpleTy != MVT::i32)
7814 return Register();
7815 if ((!Subtarget->isThumb())) {
7816 return fastEmitInst_ri(MachineInstOpcode: ARM::ADDri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7817 }
7818 return Register();
7819}
7820
7821Register fastEmit_ISD_ADD_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7822 switch (VT.SimpleTy) {
7823 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
7824 default: return Register();
7825 }
7826}
7827
7828// FastEmit functions for ISD::AND.
7829
7830Register fastEmit_ISD_AND_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
7831 if (RetVT.SimpleTy != MVT::i32)
7832 return Register();
7833 if ((!Subtarget->isThumb())) {
7834 return fastEmitInst_ri(MachineInstOpcode: ARM::ANDri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7835 }
7836 return Register();
7837}
7838
7839Register fastEmit_ISD_AND_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7840 switch (VT.SimpleTy) {
7841 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
7842 default: return Register();
7843 }
7844}
7845
7846// FastEmit functions for ISD::OR.
7847
7848Register fastEmit_ISD_OR_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
7849 if (RetVT.SimpleTy != MVT::i32)
7850 return Register();
7851 if ((!Subtarget->isThumb())) {
7852 return fastEmitInst_ri(MachineInstOpcode: ARM::ORRri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7853 }
7854 return Register();
7855}
7856
7857Register fastEmit_ISD_OR_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7858 switch (VT.SimpleTy) {
7859 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
7860 default: return Register();
7861 }
7862}
7863
7864// FastEmit functions for ISD::SUB.
7865
7866Register fastEmit_ISD_SUB_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
7867 if (RetVT.SimpleTy != MVT::i32)
7868 return Register();
7869 if ((!Subtarget->isThumb())) {
7870 return fastEmitInst_ri(MachineInstOpcode: ARM::SUBri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7871 }
7872 return Register();
7873}
7874
7875Register fastEmit_ISD_SUB_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7876 switch (VT.SimpleTy) {
7877 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
7878 default: return Register();
7879 }
7880}
7881
7882// FastEmit functions for ISD::XOR.
7883
7884Register fastEmit_ISD_XOR_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
7885 if (RetVT.SimpleTy != MVT::i32)
7886 return Register();
7887 if ((!Subtarget->isThumb())) {
7888 return fastEmitInst_ri(MachineInstOpcode: ARM::EORri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7889 }
7890 return Register();
7891}
7892
7893Register fastEmit_ISD_XOR_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7894 switch (VT.SimpleTy) {
7895 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
7896 default: return Register();
7897 }
7898}
7899
7900// Top-level FastEmit function.
7901
7902Register fastEmit_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
7903 switch (Opcode) {
7904 case ARMISD::CMN: return fastEmit_ARMISD_CMN_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
7905 case ARMISD::CMP: return fastEmit_ARMISD_CMP_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
7906 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
7907 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
7908 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
7909 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
7910 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
7911 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
7912 default: return Register();
7913 }
7914}
7915
7916// FastEmit functions for ARMISD::VSHLIMM.
7917
7918Register fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
7919 if (RetVT.SimpleTy != MVT::v16i8)
7920 return Register();
7921 if ((Subtarget->hasMVEIntegerOps())) {
7922 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHL_immi8, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
7923 }
7924 return Register();
7925}
7926
7927Register fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7928 switch (VT.SimpleTy) {
7929 case MVT::v16i8: return fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, imm1);
7930 default: return Register();
7931 }
7932}
7933
7934// FastEmit functions for ARMISD::VSHRsIMM.
7935
7936Register fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
7937 if (RetVT.SimpleTy != MVT::v16i8)
7938 return Register();
7939 if ((Subtarget->hasMVEIntegerOps())) {
7940 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_imms8, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
7941 }
7942 return Register();
7943}
7944
7945Register fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7946 switch (VT.SimpleTy) {
7947 case MVT::v16i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, imm1);
7948 default: return Register();
7949 }
7950}
7951
7952// FastEmit functions for ARMISD::VSHRuIMM.
7953
7954Register fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
7955 if (RetVT.SimpleTy != MVT::v16i8)
7956 return Register();
7957 if ((Subtarget->hasMVEIntegerOps())) {
7958 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_immu8, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
7959 }
7960 return Register();
7961}
7962
7963Register fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7964 switch (VT.SimpleTy) {
7965 case MVT::v16i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, imm1);
7966 default: return Register();
7967 }
7968}
7969
7970// FastEmit functions for ISD::ADD.
7971
7972Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
7973 if (RetVT.SimpleTy != MVT::i32)
7974 return Register();
7975 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
7976 return fastEmitInst_ri(MachineInstOpcode: ARM::tADDi3, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
7977 }
7978 return Register();
7979}
7980
7981Register fastEmit_ISD_ADD_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7982 switch (VT.SimpleTy) {
7983 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_7(RetVT, Op0, imm1);
7984 default: return Register();
7985 }
7986}
7987
7988// Top-level FastEmit function.
7989
7990Register fastEmit_ri_Predicate_imm0_7(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
7991 switch (Opcode) {
7992 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
7993 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
7994 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
7995 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
7996 default: return Register();
7997 }
7998}
7999
8000// FastEmit functions for ISD::ADD.
8001
8002Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_255_expr(MVT RetVT, Register Op0, uint64_t imm1) {
8003 if (RetVT.SimpleTy != MVT::i32)
8004 return Register();
8005 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
8006 return fastEmitInst_ri(MachineInstOpcode: ARM::tADDi8, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
8007 }
8008 return Register();
8009}
8010
8011Register fastEmit_ISD_ADD_ri_Predicate_imm0_255_expr(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8012 switch (VT.SimpleTy) {
8013 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_255_expr(RetVT, Op0, imm1);
8014 default: return Register();
8015 }
8016}
8017
8018// Top-level FastEmit function.
8019
8020Register fastEmit_ri_Predicate_imm0_255_expr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8021 switch (Opcode) {
8022 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm0_255_expr(VT, RetVT, Op0, imm1);
8023 default: return Register();
8024 }
8025}
8026
8027// FastEmit functions for ARMISD::CMP.
8028
8029Register fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_imm0_255(MVT RetVT, Register Op0, uint64_t imm1) {
8030 if (RetVT.SimpleTy != MVT::i32)
8031 return Register();
8032 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
8033 return fastEmitInst_ri(MachineInstOpcode: ARM::tCMPi8, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
8034 }
8035 return Register();
8036}
8037
8038Register fastEmit_ARMISD_CMP_ri_Predicate_imm0_255(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8039 switch (VT.SimpleTy) {
8040 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_imm0_255(RetVT, Op0, imm1);
8041 default: return Register();
8042 }
8043}
8044
8045// FastEmit functions for ARMISD::CMPZ.
8046
8047Register fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_imm0_255(MVT RetVT, Register Op0, uint64_t imm1) {
8048 if (RetVT.SimpleTy != MVT::i32)
8049 return Register();
8050 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
8051 return fastEmitInst_ri(MachineInstOpcode: ARM::tCMPi8, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
8052 }
8053 return Register();
8054}
8055
8056Register fastEmit_ARMISD_CMPZ_ri_Predicate_imm0_255(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8057 switch (VT.SimpleTy) {
8058 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_imm0_255(RetVT, Op0, imm1);
8059 default: return Register();
8060 }
8061}
8062
8063// Top-level FastEmit function.
8064
8065Register fastEmit_ri_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8066 switch (Opcode) {
8067 case ARMISD::CMP: return fastEmit_ARMISD_CMP_ri_Predicate_imm0_255(VT, RetVT, Op0, imm1);
8068 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_ri_Predicate_imm0_255(VT, RetVT, Op0, imm1);
8069 default: return Register();
8070 }
8071}
8072
8073// FastEmit functions for ARMISD::CMP.
8074
8075Register fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8076 if (RetVT.SimpleTy != MVT::i32)
8077 return Register();
8078 if ((Subtarget->isThumb2())) {
8079 return fastEmitInst_ri(MachineInstOpcode: ARM::t2CMPri, RC: &ARM::GPRnopcRegClass, Op0, Imm: imm1);
8080 }
8081 return Register();
8082}
8083
8084Register fastEmit_ARMISD_CMP_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8085 switch (VT.SimpleTy) {
8086 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8087 default: return Register();
8088 }
8089}
8090
8091// FastEmit functions for ARMISD::CMPZ.
8092
8093Register fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8094 if (RetVT.SimpleTy != MVT::i32)
8095 return Register();
8096 if ((Subtarget->isThumb2())) {
8097 return fastEmitInst_ri(MachineInstOpcode: ARM::t2CMPri, RC: &ARM::GPRnopcRegClass, Op0, Imm: imm1);
8098 }
8099 return Register();
8100}
8101
8102Register fastEmit_ARMISD_CMPZ_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8103 switch (VT.SimpleTy) {
8104 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8105 default: return Register();
8106 }
8107}
8108
8109// FastEmit functions for ISD::ADD.
8110
8111Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8112 if (RetVT.SimpleTy != MVT::i32)
8113 return Register();
8114 if ((Subtarget->isThumb2())) {
8115 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ADDri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8116 }
8117 return Register();
8118}
8119
8120Register fastEmit_ISD_ADD_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8121 switch (VT.SimpleTy) {
8122 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8123 default: return Register();
8124 }
8125}
8126
8127// FastEmit functions for ISD::AND.
8128
8129Register fastEmit_ISD_AND_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8130 if (RetVT.SimpleTy != MVT::i32)
8131 return Register();
8132 if ((Subtarget->isThumb2())) {
8133 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ANDri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8134 }
8135 return Register();
8136}
8137
8138Register fastEmit_ISD_AND_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8139 switch (VT.SimpleTy) {
8140 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8141 default: return Register();
8142 }
8143}
8144
8145// FastEmit functions for ISD::OR.
8146
8147Register fastEmit_ISD_OR_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8148 if (RetVT.SimpleTy != MVT::i32)
8149 return Register();
8150 if ((Subtarget->isThumb2())) {
8151 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ORRri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8152 }
8153 return Register();
8154}
8155
8156Register fastEmit_ISD_OR_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8157 switch (VT.SimpleTy) {
8158 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8159 default: return Register();
8160 }
8161}
8162
8163// FastEmit functions for ISD::SUB.
8164
8165Register fastEmit_ISD_SUB_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8166 if (RetVT.SimpleTy != MVT::i32)
8167 return Register();
8168 if ((Subtarget->isThumb2())) {
8169 return fastEmitInst_ri(MachineInstOpcode: ARM::t2SUBri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8170 }
8171 return Register();
8172}
8173
8174Register fastEmit_ISD_SUB_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8175 switch (VT.SimpleTy) {
8176 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8177 default: return Register();
8178 }
8179}
8180
8181// FastEmit functions for ISD::XOR.
8182
8183Register fastEmit_ISD_XOR_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8184 if (RetVT.SimpleTy != MVT::i32)
8185 return Register();
8186 if ((Subtarget->isThumb2())) {
8187 return fastEmitInst_ri(MachineInstOpcode: ARM::t2EORri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8188 }
8189 return Register();
8190}
8191
8192Register fastEmit_ISD_XOR_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8193 switch (VT.SimpleTy) {
8194 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8195 default: return Register();
8196 }
8197}
8198
8199// Top-level FastEmit function.
8200
8201Register fastEmit_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8202 switch (Opcode) {
8203 case ARMISD::CMP: return fastEmit_ARMISD_CMP_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8204 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8205 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8206 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8207 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8208 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8209 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8210 default: return Register();
8211 }
8212}
8213
8214// FastEmit functions for ISD::ADD.
8215
8216Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_4095(MVT RetVT, Register Op0, uint64_t imm1) {
8217 if (RetVT.SimpleTy != MVT::i32)
8218 return Register();
8219 if ((Subtarget->isThumb2())) {
8220 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ADDri12, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8221 }
8222 return Register();
8223}
8224
8225Register fastEmit_ISD_ADD_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8226 switch (VT.SimpleTy) {
8227 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_4095(RetVT, Op0, imm1);
8228 default: return Register();
8229 }
8230}
8231
8232// FastEmit functions for ISD::SUB.
8233
8234Register fastEmit_ISD_SUB_MVT_i32_ri_Predicate_imm0_4095(MVT RetVT, Register Op0, uint64_t imm1) {
8235 if (RetVT.SimpleTy != MVT::i32)
8236 return Register();
8237 if ((Subtarget->isThumb2())) {
8238 return fastEmitInst_ri(MachineInstOpcode: ARM::t2SUBri12, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8239 }
8240 return Register();
8241}
8242
8243Register fastEmit_ISD_SUB_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8244 switch (VT.SimpleTy) {
8245 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_imm0_4095(RetVT, Op0, imm1);
8246 default: return Register();
8247 }
8248}
8249
8250// Top-level FastEmit function.
8251
8252Register fastEmit_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8253 switch (Opcode) {
8254 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm0_4095(VT, RetVT, Op0, imm1);
8255 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_imm0_4095(VT, RetVT, Op0, imm1);
8256 default: return Register();
8257 }
8258}
8259
8260// FastEmit functions for ISD::ROTR.
8261
8262Register fastEmit_ISD_ROTR_MVT_i32_ri_Predicate_imm1_31(MVT RetVT, Register Op0, uint64_t imm1) {
8263 if (RetVT.SimpleTy != MVT::i32)
8264 return Register();
8265 if ((Subtarget->isThumb2())) {
8266 return fastEmitInst_ri(MachineInstOpcode: ARM::t2RORri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8267 }
8268 return Register();
8269}
8270
8271Register fastEmit_ISD_ROTR_ri_Predicate_imm1_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8272 switch (VT.SimpleTy) {
8273 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_ri_Predicate_imm1_31(RetVT, Op0, imm1);
8274 default: return Register();
8275 }
8276}
8277
8278// FastEmit functions for ISD::SHL.
8279
8280Register fastEmit_ISD_SHL_MVT_i32_ri_Predicate_imm1_31(MVT RetVT, Register Op0, uint64_t imm1) {
8281 if (RetVT.SimpleTy != MVT::i32)
8282 return Register();
8283 if ((Subtarget->isThumb2())) {
8284 return fastEmitInst_ri(MachineInstOpcode: ARM::t2LSLri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8285 }
8286 return Register();
8287}
8288
8289Register fastEmit_ISD_SHL_ri_Predicate_imm1_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8290 switch (VT.SimpleTy) {
8291 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri_Predicate_imm1_31(RetVT, Op0, imm1);
8292 default: return Register();
8293 }
8294}
8295
8296// Top-level FastEmit function.
8297
8298Register fastEmit_ri_Predicate_imm1_31(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8299 switch (Opcode) {
8300 case ISD::ROTR: return fastEmit_ISD_ROTR_ri_Predicate_imm1_31(VT, RetVT, Op0, imm1);
8301 case ISD::SHL: return fastEmit_ISD_SHL_ri_Predicate_imm1_31(VT, RetVT, Op0, imm1);
8302 default: return Register();
8303 }
8304}
8305
8306// FastEmit functions for ARMISD::VQRSHRNsIMM.
8307
8308Register fastEmit_ARMISD_VQRSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8309 if (RetVT.SimpleTy != MVT::v8i8)
8310 return Register();
8311 if ((Subtarget->hasNEON())) {
8312 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8313 }
8314 return Register();
8315}
8316
8317Register fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8318 switch (VT.SimpleTy) {
8319 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8320 default: return Register();
8321 }
8322}
8323
8324// FastEmit functions for ARMISD::VQRSHRNsuIMM.
8325
8326Register fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8327 if (RetVT.SimpleTy != MVT::v8i8)
8328 return Register();
8329 if ((Subtarget->hasNEON())) {
8330 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRUNv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8331 }
8332 return Register();
8333}
8334
8335Register fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8336 switch (VT.SimpleTy) {
8337 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8338 default: return Register();
8339 }
8340}
8341
8342// FastEmit functions for ARMISD::VQRSHRNuIMM.
8343
8344Register fastEmit_ARMISD_VQRSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8345 if (RetVT.SimpleTy != MVT::v8i8)
8346 return Register();
8347 if ((Subtarget->hasNEON())) {
8348 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8349 }
8350 return Register();
8351}
8352
8353Register fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8354 switch (VT.SimpleTy) {
8355 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8356 default: return Register();
8357 }
8358}
8359
8360// FastEmit functions for ARMISD::VQSHRNsIMM.
8361
8362Register fastEmit_ARMISD_VQSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8363 if (RetVT.SimpleTy != MVT::v8i8)
8364 return Register();
8365 if ((Subtarget->hasNEON())) {
8366 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8367 }
8368 return Register();
8369}
8370
8371Register fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8372 switch (VT.SimpleTy) {
8373 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8374 default: return Register();
8375 }
8376}
8377
8378// FastEmit functions for ARMISD::VQSHRNsuIMM.
8379
8380Register fastEmit_ARMISD_VQSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8381 if (RetVT.SimpleTy != MVT::v8i8)
8382 return Register();
8383 if ((Subtarget->hasNEON())) {
8384 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRUNv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8385 }
8386 return Register();
8387}
8388
8389Register fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8390 switch (VT.SimpleTy) {
8391 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8392 default: return Register();
8393 }
8394}
8395
8396// FastEmit functions for ARMISD::VQSHRNuIMM.
8397
8398Register fastEmit_ARMISD_VQSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8399 if (RetVT.SimpleTy != MVT::v8i8)
8400 return Register();
8401 if ((Subtarget->hasNEON())) {
8402 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8403 }
8404 return Register();
8405}
8406
8407Register fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8408 switch (VT.SimpleTy) {
8409 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8410 default: return Register();
8411 }
8412}
8413
8414// FastEmit functions for ARMISD::VRSHRNIMM.
8415
8416Register fastEmit_ARMISD_VRSHRNIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
8417 if (RetVT.SimpleTy != MVT::v8i8)
8418 return Register();
8419 if ((Subtarget->hasNEON())) {
8420 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRNv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8421 }
8422 return Register();
8423}
8424
8425Register fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8426 switch (VT.SimpleTy) {
8427 case MVT::v8i16: return fastEmit_ARMISD_VRSHRNIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8428 default: return Register();
8429 }
8430}
8431
8432// Top-level FastEmit function.
8433
8434Register fastEmit_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8435 switch (Opcode) {
8436 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8437 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8438 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8439 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8440 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8441 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8442 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8443 default: return Register();
8444 }
8445}
8446
8447// FastEmit functions for ARMISD::VQRSHRNsIMM.
8448
8449Register fastEmit_ARMISD_VQRSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
8450 if (RetVT.SimpleTy != MVT::v4i16)
8451 return Register();
8452 if ((Subtarget->hasNEON())) {
8453 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8454 }
8455 return Register();
8456}
8457
8458Register fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8459 switch (VT.SimpleTy) {
8460 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8461 default: return Register();
8462 }
8463}
8464
8465// FastEmit functions for ARMISD::VQRSHRNsuIMM.
8466
8467Register fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
8468 if (RetVT.SimpleTy != MVT::v4i16)
8469 return Register();
8470 if ((Subtarget->hasNEON())) {
8471 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRUNv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8472 }
8473 return Register();
8474}
8475
8476Register fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8477 switch (VT.SimpleTy) {
8478 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8479 default: return Register();
8480 }
8481}
8482
8483// FastEmit functions for ARMISD::VQRSHRNuIMM.
8484
8485Register fastEmit_ARMISD_VQRSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
8486 if (RetVT.SimpleTy != MVT::v4i16)
8487 return Register();
8488 if ((Subtarget->hasNEON())) {
8489 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8490 }
8491 return Register();
8492}
8493
8494Register fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8495 switch (VT.SimpleTy) {
8496 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8497 default: return Register();
8498 }
8499}
8500
8501// FastEmit functions for ARMISD::VQSHRNsIMM.
8502
8503Register fastEmit_ARMISD_VQSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
8504 if (RetVT.SimpleTy != MVT::v4i16)
8505 return Register();
8506 if ((Subtarget->hasNEON())) {
8507 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8508 }
8509 return Register();
8510}
8511
8512Register fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8513 switch (VT.SimpleTy) {
8514 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8515 default: return Register();
8516 }
8517}
8518
8519// FastEmit functions for ARMISD::VQSHRNsuIMM.
8520
8521Register fastEmit_ARMISD_VQSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
8522 if (RetVT.SimpleTy != MVT::v4i16)
8523 return Register();
8524 if ((Subtarget->hasNEON())) {
8525 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRUNv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8526 }
8527 return Register();
8528}
8529
8530Register fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8531 switch (VT.SimpleTy) {
8532 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8533 default: return Register();
8534 }
8535}
8536
8537// FastEmit functions for ARMISD::VQSHRNuIMM.
8538
8539Register fastEmit_ARMISD_VQSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
8540 if (RetVT.SimpleTy != MVT::v4i16)
8541 return Register();
8542 if ((Subtarget->hasNEON())) {
8543 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8544 }
8545 return Register();
8546}
8547
8548Register fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8549 switch (VT.SimpleTy) {
8550 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8551 default: return Register();
8552 }
8553}
8554
8555// FastEmit functions for ARMISD::VRSHRNIMM.
8556
8557Register fastEmit_ARMISD_VRSHRNIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
8558 if (RetVT.SimpleTy != MVT::v4i16)
8559 return Register();
8560 if ((Subtarget->hasNEON())) {
8561 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRNv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8562 }
8563 return Register();
8564}
8565
8566Register fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8567 switch (VT.SimpleTy) {
8568 case MVT::v4i32: return fastEmit_ARMISD_VRSHRNIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8569 default: return Register();
8570 }
8571}
8572
8573// Top-level FastEmit function.
8574
8575Register fastEmit_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8576 switch (Opcode) {
8577 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
8578 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
8579 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
8580 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
8581 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
8582 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
8583 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
8584 default: return Register();
8585 }
8586}
8587
8588// FastEmit functions for ARMISD::VQRSHRNsIMM.
8589
8590Register fastEmit_ARMISD_VQRSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
8591 if (RetVT.SimpleTy != MVT::v2i32)
8592 return Register();
8593 if ((Subtarget->hasNEON())) {
8594 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8595 }
8596 return Register();
8597}
8598
8599Register fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8600 switch (VT.SimpleTy) {
8601 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
8602 default: return Register();
8603 }
8604}
8605
8606// FastEmit functions for ARMISD::VQRSHRNsuIMM.
8607
8608Register fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
8609 if (RetVT.SimpleTy != MVT::v2i32)
8610 return Register();
8611 if ((Subtarget->hasNEON())) {
8612 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRUNv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8613 }
8614 return Register();
8615}
8616
8617Register fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8618 switch (VT.SimpleTy) {
8619 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
8620 default: return Register();
8621 }
8622}
8623
8624// FastEmit functions for ARMISD::VQRSHRNuIMM.
8625
8626Register fastEmit_ARMISD_VQRSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
8627 if (RetVT.SimpleTy != MVT::v2i32)
8628 return Register();
8629 if ((Subtarget->hasNEON())) {
8630 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8631 }
8632 return Register();
8633}
8634
8635Register fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8636 switch (VT.SimpleTy) {
8637 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
8638 default: return Register();
8639 }
8640}
8641
8642// FastEmit functions for ARMISD::VQSHRNsIMM.
8643
8644Register fastEmit_ARMISD_VQSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
8645 if (RetVT.SimpleTy != MVT::v2i32)
8646 return Register();
8647 if ((Subtarget->hasNEON())) {
8648 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8649 }
8650 return Register();
8651}
8652
8653Register fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8654 switch (VT.SimpleTy) {
8655 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
8656 default: return Register();
8657 }
8658}
8659
8660// FastEmit functions for ARMISD::VQSHRNsuIMM.
8661
8662Register fastEmit_ARMISD_VQSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
8663 if (RetVT.SimpleTy != MVT::v2i32)
8664 return Register();
8665 if ((Subtarget->hasNEON())) {
8666 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRUNv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8667 }
8668 return Register();
8669}
8670
8671Register fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8672 switch (VT.SimpleTy) {
8673 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
8674 default: return Register();
8675 }
8676}
8677
8678// FastEmit functions for ARMISD::VQSHRNuIMM.
8679
8680Register fastEmit_ARMISD_VQSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
8681 if (RetVT.SimpleTy != MVT::v2i32)
8682 return Register();
8683 if ((Subtarget->hasNEON())) {
8684 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8685 }
8686 return Register();
8687}
8688
8689Register fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8690 switch (VT.SimpleTy) {
8691 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
8692 default: return Register();
8693 }
8694}
8695
8696// FastEmit functions for ARMISD::VRSHRNIMM.
8697
8698Register fastEmit_ARMISD_VRSHRNIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
8699 if (RetVT.SimpleTy != MVT::v2i32)
8700 return Register();
8701 if ((Subtarget->hasNEON())) {
8702 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRNv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8703 }
8704 return Register();
8705}
8706
8707Register fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8708 switch (VT.SimpleTy) {
8709 case MVT::v2i64: return fastEmit_ARMISD_VRSHRNIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
8710 default: return Register();
8711 }
8712}
8713
8714// Top-level FastEmit function.
8715
8716Register fastEmit_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8717 switch (Opcode) {
8718 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
8719 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
8720 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
8721 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
8722 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
8723 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
8724 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
8725 default: return Register();
8726 }
8727}
8728
8729// FastEmit functions for ARMISD::VDUPLANE.
8730
8731Register fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri_Predicate_VectorIndex32(MVT RetVT, Register Op0, uint64_t imm1) {
8732 if (RetVT.SimpleTy != MVT::v16i8)
8733 return Register();
8734 if ((Subtarget->hasNEON())) {
8735 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN8q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8736 }
8737 return Register();
8738}
8739
8740Register fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri_Predicate_VectorIndex32(MVT RetVT, Register Op0, uint64_t imm1) {
8741 if (RetVT.SimpleTy != MVT::v8i16)
8742 return Register();
8743 if ((Subtarget->hasNEON())) {
8744 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8745 }
8746 return Register();
8747}
8748
8749Register fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri_Predicate_VectorIndex32(MVT RetVT, Register Op0, uint64_t imm1) {
8750 if (RetVT.SimpleTy != MVT::v4i32)
8751 return Register();
8752 if ((Subtarget->hasNEON())) {
8753 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8754 }
8755 return Register();
8756}
8757
8758Register fastEmit_ARMISD_VDUPLANE_ri_Predicate_VectorIndex32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8759 switch (VT.SimpleTy) {
8760 case MVT::v8i8: return fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri_Predicate_VectorIndex32(RetVT, Op0, imm1);
8761 case MVT::v4i16: return fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri_Predicate_VectorIndex32(RetVT, Op0, imm1);
8762 case MVT::v2i32: return fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri_Predicate_VectorIndex32(RetVT, Op0, imm1);
8763 default: return Register();
8764 }
8765}
8766
8767// Top-level FastEmit function.
8768
8769Register fastEmit_ri_Predicate_VectorIndex32(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8770 switch (Opcode) {
8771 case ARMISD::VDUPLANE: return fastEmit_ARMISD_VDUPLANE_ri_Predicate_VectorIndex32(VT, RetVT, Op0, imm1);
8772 default: return Register();
8773 }
8774}
8775
8776// FastEmit functions for ARMISD::VSHLIMM.
8777
8778Register fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, Register Op0, uint64_t imm1) {
8779 if (RetVT.SimpleTy != MVT::v4i32)
8780 return Register();
8781 if ((Subtarget->hasMVEIntegerOps())) {
8782 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHL_immi32, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8783 }
8784 return Register();
8785}
8786
8787Register fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8788 switch (VT.SimpleTy) {
8789 case MVT::v4i32: return fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, imm1);
8790 default: return Register();
8791 }
8792}
8793
8794// FastEmit functions for ARMISD::VSHRsIMM.
8795
8796Register fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, Register Op0, uint64_t imm1) {
8797 if (RetVT.SimpleTy != MVT::v4i32)
8798 return Register();
8799 if ((Subtarget->hasMVEIntegerOps())) {
8800 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_imms32, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8801 }
8802 return Register();
8803}
8804
8805Register fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8806 switch (VT.SimpleTy) {
8807 case MVT::v4i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, imm1);
8808 default: return Register();
8809 }
8810}
8811
8812// FastEmit functions for ARMISD::VSHRuIMM.
8813
8814Register fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, Register Op0, uint64_t imm1) {
8815 if (RetVT.SimpleTy != MVT::v4i32)
8816 return Register();
8817 if ((Subtarget->hasMVEIntegerOps())) {
8818 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_immu32, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8819 }
8820 return Register();
8821}
8822
8823Register fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8824 switch (VT.SimpleTy) {
8825 case MVT::v4i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, imm1);
8826 default: return Register();
8827 }
8828}
8829
8830// Top-level FastEmit function.
8831
8832Register fastEmit_ri_Predicate_imm0_31(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8833 switch (Opcode) {
8834 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, imm1);
8835 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, imm1);
8836 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, imm1);
8837 default: return Register();
8838 }
8839}
8840
8841// FastEmit functions for ARMISD::VSHLIMM.
8842
8843Register fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, Register Op0, uint64_t imm1) {
8844 if (RetVT.SimpleTy != MVT::v8i16)
8845 return Register();
8846 if ((Subtarget->hasMVEIntegerOps())) {
8847 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHL_immi16, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8848 }
8849 return Register();
8850}
8851
8852Register fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8853 switch (VT.SimpleTy) {
8854 case MVT::v8i16: return fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, imm1);
8855 default: return Register();
8856 }
8857}
8858
8859// FastEmit functions for ARMISD::VSHRsIMM.
8860
8861Register fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, Register Op0, uint64_t imm1) {
8862 if (RetVT.SimpleTy != MVT::v8i16)
8863 return Register();
8864 if ((Subtarget->hasMVEIntegerOps())) {
8865 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_imms16, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8866 }
8867 return Register();
8868}
8869
8870Register fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8871 switch (VT.SimpleTy) {
8872 case MVT::v8i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, imm1);
8873 default: return Register();
8874 }
8875}
8876
8877// FastEmit functions for ARMISD::VSHRuIMM.
8878
8879Register fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, Register Op0, uint64_t imm1) {
8880 if (RetVT.SimpleTy != MVT::v8i16)
8881 return Register();
8882 if ((Subtarget->hasMVEIntegerOps())) {
8883 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_immu16, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8884 }
8885 return Register();
8886}
8887
8888Register fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8889 switch (VT.SimpleTy) {
8890 case MVT::v8i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, imm1);
8891 default: return Register();
8892 }
8893}
8894
8895// Top-level FastEmit function.
8896
8897Register fastEmit_ri_Predicate_imm0_15(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8898 switch (Opcode) {
8899 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, imm1);
8900 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, imm1);
8901 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, imm1);
8902 default: return Register();
8903 }
8904}
8905
8906// FastEmit functions for ISD::Constant.
8907
8908Register fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
8909 if (RetVT.SimpleTy != MVT::i32)
8910 return Register();
8911 if ((Subtarget->isThumb()) && (Subtarget->useMovt())) {
8912 return fastEmitInst_i(MachineInstOpcode: ARM::t2MOVi32imm, RC: &ARM::rGPRRegClass, Imm: imm0);
8913 }
8914 if ((!Subtarget->useMovt()) && (Subtarget->genExecuteOnly()) && (Subtarget->isThumb1Only())) {
8915 return fastEmitInst_i(MachineInstOpcode: ARM::tMOVi32imm, RC: &ARM::rGPRRegClass, Imm: imm0);
8916 }
8917 return Register();
8918}
8919
8920Register fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
8921 switch (VT.SimpleTy) {
8922 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
8923 default: return Register();
8924 }
8925}
8926
8927// Top-level FastEmit function.
8928
8929Register fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
8930 switch (Opcode) {
8931 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
8932 default: return Register();
8933 }
8934}
8935
8936