1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the ARM target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_mod_imm(int64_t Imm) {
12
13 return ARM_AM::getSOImmVal(Arg: Imm) != -1;
14
15}
16static bool Predicate_imm0_65535(int64_t Imm) {
17
18 return Imm >= 0 && Imm < 65536;
19
20}
21static bool Predicate_mod_imm_not(int64_t Imm) {
22
23 return ARM_AM::getSOImmVal(Arg: ~(uint32_t)Imm) != -1;
24
25}
26static bool Predicate_imm0_7(int64_t Imm) {
27
28 return Imm >= 0 && Imm < 8;
29
30}
31static bool Predicate_imm0_255_expr(int64_t Imm) {
32 return Imm >= 0 && Imm < 256;
33}
34static bool Predicate_imm0_255(int64_t Imm) {
35 return Imm >= 0 && Imm < 256;
36}
37static bool Predicate_t2_so_imm(int64_t Imm) {
38
39 return ARM_AM::getT2SOImmVal(Arg: Imm) != -1;
40
41}
42static bool Predicate_imm0_4095(int64_t Imm) {
43
44 return Imm >= 0 && Imm < 4096;
45
46}
47static bool Predicate_imm1_31(int64_t Imm) {
48 return Imm > 0 && Imm < 32;
49}
50static bool Predicate_shr_imm8(int64_t Imm) {
51 return Imm > 0 && Imm <= 8;
52}
53static bool Predicate_shr_imm16(int64_t Imm) {
54 return Imm > 0 && Imm <= 16;
55}
56static bool Predicate_shr_imm32(int64_t Imm) {
57 return Imm > 0 && Imm <= 32;
58}
59static bool Predicate_VectorIndex32(int64_t Imm) {
60
61 return ((uint64_t)Imm) < 2;
62
63}
64static bool Predicate_imm0_31(int64_t Imm) {
65
66 return Imm >= 0 && Imm < 32;
67
68}
69static bool Predicate_t2_so_imm_neg(int64_t Imm) {
70
71 return Imm && ARM_AM::getT2SOImmVal(Arg: -(uint32_t)Imm) != -1;
72
73}
74static bool Predicate_imm0_15(int64_t Imm) {
75
76 return Imm >= 0 && Imm < 16;
77
78}
79
80
81// FastEmit functions for ISD::GET_FPENV.
82
83unsigned fastEmit_ISD_GET_FPENV_MVT_i32_(MVT RetVT) {
84 if (RetVT.SimpleTy != MVT::i32)
85 return 0;
86 return fastEmitInst_(MachineInstOpcode: ARM::VMRS, RC: &ARM::GPRnopcRegClass);
87}
88
89unsigned fastEmit_ISD_GET_FPENV_(MVT VT, MVT RetVT) {
90 switch (VT.SimpleTy) {
91 case MVT::i32: return fastEmit_ISD_GET_FPENV_MVT_i32_(RetVT);
92 default: return 0;
93 }
94}
95
96// FastEmit functions for ISD::GET_FPMODE.
97
98unsigned fastEmit_ISD_GET_FPMODE_MVT_i32_(MVT RetVT) {
99 if (RetVT.SimpleTy != MVT::i32)
100 return 0;
101 return fastEmitInst_(MachineInstOpcode: ARM::VMRS, RC: &ARM::GPRnopcRegClass);
102}
103
104unsigned fastEmit_ISD_GET_FPMODE_(MVT VT, MVT RetVT) {
105 switch (VT.SimpleTy) {
106 case MVT::i32: return fastEmit_ISD_GET_FPMODE_MVT_i32_(RetVT);
107 default: return 0;
108 }
109}
110
111// Top-level FastEmit function.
112
113unsigned fastEmit_(MVT VT, MVT RetVT, unsigned Opcode) override {
114 switch (Opcode) {
115 case ISD::GET_FPENV: return fastEmit_ISD_GET_FPENV_(VT, RetVT);
116 case ISD::GET_FPMODE: return fastEmit_ISD_GET_FPMODE_(VT, RetVT);
117 default: return 0;
118 }
119}
120
121// FastEmit functions for ARMISD::CALL.
122
123unsigned fastEmit_ARMISD_CALL_MVT_i32_r(MVT RetVT, unsigned Op0) {
124 if (RetVT.SimpleTy != MVT::isVoid)
125 return 0;
126 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
127 return fastEmitInst_r(MachineInstOpcode: ARM::BLX_noip, RC: &ARM::GPRnoipRegClass, Op0);
128 }
129 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( !MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
130 return fastEmitInst_r(MachineInstOpcode: ARM::BLX, RC: &ARM::GPRRegClass, Op0);
131 }
132 return 0;
133}
134
135unsigned fastEmit_ARMISD_CALL_r(MVT VT, MVT RetVT, unsigned Op0) {
136 switch (VT.SimpleTy) {
137 case MVT::i32: return fastEmit_ARMISD_CALL_MVT_i32_r(RetVT, Op0);
138 default: return 0;
139 }
140}
141
142// FastEmit functions for ARMISD::CALL_NOLINK.
143
144unsigned fastEmit_ARMISD_CALL_NOLINK_MVT_i32_r(MVT RetVT, unsigned Op0) {
145 if (RetVT.SimpleTy != MVT::isVoid)
146 return 0;
147 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
148 return fastEmitInst_r(MachineInstOpcode: ARM::tBX_CALL, RC: &ARM::tGPRRegClass, Op0);
149 }
150 if ((!Subtarget->isThumb()) && (!Subtarget->hasV4TOps())) {
151 return fastEmitInst_r(MachineInstOpcode: ARM::BMOVPCRX_CALL, RC: &ARM::tGPRRegClass, Op0);
152 }
153 if ((Subtarget->hasV4TOps()) && (!Subtarget->isThumb())) {
154 return fastEmitInst_r(MachineInstOpcode: ARM::BX_CALL, RC: &ARM::tGPRRegClass, Op0);
155 }
156 return 0;
157}
158
159unsigned fastEmit_ARMISD_CALL_NOLINK_r(MVT VT, MVT RetVT, unsigned Op0) {
160 switch (VT.SimpleTy) {
161 case MVT::i32: return fastEmit_ARMISD_CALL_NOLINK_MVT_i32_r(RetVT, Op0);
162 default: return 0;
163 }
164}
165
166// FastEmit functions for ARMISD::CALL_PRED.
167
168unsigned fastEmit_ARMISD_CALL_PRED_MVT_i32_r(MVT RetVT, unsigned Op0) {
169 if (RetVT.SimpleTy != MVT::isVoid)
170 return 0;
171 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
172 return fastEmitInst_r(MachineInstOpcode: ARM::BLX_pred_noip, RC: &ARM::GPRnoipRegClass, Op0);
173 }
174 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( !MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
175 return fastEmitInst_r(MachineInstOpcode: ARM::BLX_pred, RC: &ARM::GPRRegClass, Op0);
176 }
177 return 0;
178}
179
180unsigned fastEmit_ARMISD_CALL_PRED_r(MVT VT, MVT RetVT, unsigned Op0) {
181 switch (VT.SimpleTy) {
182 case MVT::i32: return fastEmit_ARMISD_CALL_PRED_MVT_i32_r(RetVT, Op0);
183 default: return 0;
184 }
185}
186
187// FastEmit functions for ARMISD::CMPFPEw0.
188
189unsigned fastEmit_ARMISD_CMPFPEw0_MVT_f16_r(MVT RetVT, unsigned Op0) {
190 if (RetVT.SimpleTy != MVT::isVoid)
191 return 0;
192 if ((Subtarget->hasFullFP16())) {
193 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPEZH, RC: &ARM::HPRRegClass, Op0);
194 }
195 return 0;
196}
197
198unsigned fastEmit_ARMISD_CMPFPEw0_MVT_f32_r(MVT RetVT, unsigned Op0) {
199 if (RetVT.SimpleTy != MVT::isVoid)
200 return 0;
201 if ((Subtarget->hasVFP2Base())) {
202 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPEZS, RC: &ARM::SPRRegClass, Op0);
203 }
204 return 0;
205}
206
207unsigned fastEmit_ARMISD_CMPFPEw0_MVT_f64_r(MVT RetVT, unsigned Op0) {
208 if (RetVT.SimpleTy != MVT::isVoid)
209 return 0;
210 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
211 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPEZD, RC: &ARM::DPRRegClass, Op0);
212 }
213 return 0;
214}
215
216unsigned fastEmit_ARMISD_CMPFPEw0_r(MVT VT, MVT RetVT, unsigned Op0) {
217 switch (VT.SimpleTy) {
218 case MVT::f16: return fastEmit_ARMISD_CMPFPEw0_MVT_f16_r(RetVT, Op0);
219 case MVT::f32: return fastEmit_ARMISD_CMPFPEw0_MVT_f32_r(RetVT, Op0);
220 case MVT::f64: return fastEmit_ARMISD_CMPFPEw0_MVT_f64_r(RetVT, Op0);
221 default: return 0;
222 }
223}
224
225// FastEmit functions for ARMISD::CMPFPw0.
226
227unsigned fastEmit_ARMISD_CMPFPw0_MVT_f16_r(MVT RetVT, unsigned Op0) {
228 if (RetVT.SimpleTy != MVT::isVoid)
229 return 0;
230 if ((Subtarget->hasFullFP16())) {
231 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPZH, RC: &ARM::HPRRegClass, Op0);
232 }
233 return 0;
234}
235
236unsigned fastEmit_ARMISD_CMPFPw0_MVT_f32_r(MVT RetVT, unsigned Op0) {
237 if (RetVT.SimpleTy != MVT::isVoid)
238 return 0;
239 if ((Subtarget->hasVFP2Base())) {
240 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPZS, RC: &ARM::SPRRegClass, Op0);
241 }
242 return 0;
243}
244
245unsigned fastEmit_ARMISD_CMPFPw0_MVT_f64_r(MVT RetVT, unsigned Op0) {
246 if (RetVT.SimpleTy != MVT::isVoid)
247 return 0;
248 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
249 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPZD, RC: &ARM::DPRRegClass, Op0);
250 }
251 return 0;
252}
253
254unsigned fastEmit_ARMISD_CMPFPw0_r(MVT VT, MVT RetVT, unsigned Op0) {
255 switch (VT.SimpleTy) {
256 case MVT::f16: return fastEmit_ARMISD_CMPFPw0_MVT_f16_r(RetVT, Op0);
257 case MVT::f32: return fastEmit_ARMISD_CMPFPw0_MVT_f32_r(RetVT, Op0);
258 case MVT::f64: return fastEmit_ARMISD_CMPFPw0_MVT_f64_r(RetVT, Op0);
259 default: return 0;
260 }
261}
262
263// FastEmit functions for ARMISD::RRX.
264
265unsigned fastEmit_ARMISD_RRX_MVT_i32_r(MVT RetVT, unsigned Op0) {
266 if (RetVT.SimpleTy != MVT::i32)
267 return 0;
268 if ((Subtarget->isThumb2())) {
269 return fastEmitInst_r(MachineInstOpcode: ARM::t2RRX, RC: &ARM::rGPRRegClass, Op0);
270 }
271 if ((!Subtarget->isThumb())) {
272 return fastEmitInst_r(MachineInstOpcode: ARM::RRX, RC: &ARM::GPRRegClass, Op0);
273 }
274 return 0;
275}
276
277unsigned fastEmit_ARMISD_RRX_r(MVT VT, MVT RetVT, unsigned Op0) {
278 switch (VT.SimpleTy) {
279 case MVT::i32: return fastEmit_ARMISD_RRX_MVT_i32_r(RetVT, Op0);
280 default: return 0;
281 }
282}
283
284// FastEmit functions for ARMISD::SRA_GLUE.
285
286unsigned fastEmit_ARMISD_SRA_GLUE_MVT_i32_r(MVT RetVT, unsigned Op0) {
287 if (RetVT.SimpleTy != MVT::i32)
288 return 0;
289 if ((Subtarget->isThumb2())) {
290 return fastEmitInst_r(MachineInstOpcode: ARM::t2MOVsra_glue, RC: &ARM::rGPRRegClass, Op0);
291 }
292 if ((!Subtarget->isThumb())) {
293 return fastEmitInst_r(MachineInstOpcode: ARM::MOVsra_glue, RC: &ARM::GPRRegClass, Op0);
294 }
295 return 0;
296}
297
298unsigned fastEmit_ARMISD_SRA_GLUE_r(MVT VT, MVT RetVT, unsigned Op0) {
299 switch (VT.SimpleTy) {
300 case MVT::i32: return fastEmit_ARMISD_SRA_GLUE_MVT_i32_r(RetVT, Op0);
301 default: return 0;
302 }
303}
304
305// FastEmit functions for ARMISD::SRL_GLUE.
306
307unsigned fastEmit_ARMISD_SRL_GLUE_MVT_i32_r(MVT RetVT, unsigned Op0) {
308 if (RetVT.SimpleTy != MVT::i32)
309 return 0;
310 if ((Subtarget->isThumb2())) {
311 return fastEmitInst_r(MachineInstOpcode: ARM::t2MOVsrl_glue, RC: &ARM::rGPRRegClass, Op0);
312 }
313 if ((!Subtarget->isThumb())) {
314 return fastEmitInst_r(MachineInstOpcode: ARM::MOVsrl_glue, RC: &ARM::GPRRegClass, Op0);
315 }
316 return 0;
317}
318
319unsigned fastEmit_ARMISD_SRL_GLUE_r(MVT VT, MVT RetVT, unsigned Op0) {
320 switch (VT.SimpleTy) {
321 case MVT::i32: return fastEmit_ARMISD_SRL_GLUE_MVT_i32_r(RetVT, Op0);
322 default: return 0;
323 }
324}
325
326// FastEmit functions for ARMISD::VADDVs.
327
328unsigned fastEmit_ARMISD_VADDVs_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
329 if (RetVT.SimpleTy != MVT::i32)
330 return 0;
331 if ((Subtarget->hasMVEIntegerOps())) {
332 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVs8no_acc, RC: &ARM::tGPREvenRegClass, Op0);
333 }
334 return 0;
335}
336
337unsigned fastEmit_ARMISD_VADDVs_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
338 if (RetVT.SimpleTy != MVT::i32)
339 return 0;
340 if ((Subtarget->hasMVEIntegerOps())) {
341 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVs16no_acc, RC: &ARM::tGPREvenRegClass, Op0);
342 }
343 return 0;
344}
345
346unsigned fastEmit_ARMISD_VADDVs_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
347 if (RetVT.SimpleTy != MVT::i32)
348 return 0;
349 if ((Subtarget->hasMVEIntegerOps())) {
350 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVs32no_acc, RC: &ARM::tGPREvenRegClass, Op0);
351 }
352 return 0;
353}
354
355unsigned fastEmit_ARMISD_VADDVs_r(MVT VT, MVT RetVT, unsigned Op0) {
356 switch (VT.SimpleTy) {
357 case MVT::v16i8: return fastEmit_ARMISD_VADDVs_MVT_v16i8_r(RetVT, Op0);
358 case MVT::v8i16: return fastEmit_ARMISD_VADDVs_MVT_v8i16_r(RetVT, Op0);
359 case MVT::v4i32: return fastEmit_ARMISD_VADDVs_MVT_v4i32_r(RetVT, Op0);
360 default: return 0;
361 }
362}
363
364// FastEmit functions for ARMISD::VADDVu.
365
366unsigned fastEmit_ARMISD_VADDVu_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
367 if (RetVT.SimpleTy != MVT::i32)
368 return 0;
369 if ((Subtarget->hasMVEIntegerOps())) {
370 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu8no_acc, RC: &ARM::tGPREvenRegClass, Op0);
371 }
372 return 0;
373}
374
375unsigned fastEmit_ARMISD_VADDVu_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
376 if (RetVT.SimpleTy != MVT::i32)
377 return 0;
378 if ((Subtarget->hasMVEIntegerOps())) {
379 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu16no_acc, RC: &ARM::tGPREvenRegClass, Op0);
380 }
381 return 0;
382}
383
384unsigned fastEmit_ARMISD_VADDVu_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
385 if (RetVT.SimpleTy != MVT::i32)
386 return 0;
387 if ((Subtarget->hasMVEIntegerOps())) {
388 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu32no_acc, RC: &ARM::tGPREvenRegClass, Op0);
389 }
390 return 0;
391}
392
393unsigned fastEmit_ARMISD_VADDVu_r(MVT VT, MVT RetVT, unsigned Op0) {
394 switch (VT.SimpleTy) {
395 case MVT::v16i8: return fastEmit_ARMISD_VADDVu_MVT_v16i8_r(RetVT, Op0);
396 case MVT::v8i16: return fastEmit_ARMISD_VADDVu_MVT_v8i16_r(RetVT, Op0);
397 case MVT::v4i32: return fastEmit_ARMISD_VADDVu_MVT_v4i32_r(RetVT, Op0);
398 default: return 0;
399 }
400}
401
402// FastEmit functions for ARMISD::VDUP.
403
404unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i8_r(unsigned Op0) {
405 if ((Subtarget->hasNEON())) {
406 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP8d, RC: &ARM::DPRRegClass, Op0);
407 }
408 return 0;
409}
410
411unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v16i8_r(unsigned Op0) {
412 if ((Subtarget->hasMVEIntegerOps())) {
413 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP8, RC: &ARM::MQPRRegClass, Op0);
414 }
415 if ((Subtarget->hasNEON())) {
416 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP8q, RC: &ARM::QPRRegClass, Op0);
417 }
418 return 0;
419}
420
421unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i16_r(unsigned Op0) {
422 if ((Subtarget->hasNEON())) {
423 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP16d, RC: &ARM::DPRRegClass, Op0);
424 }
425 return 0;
426}
427
428unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i16_r(unsigned Op0) {
429 if ((Subtarget->hasMVEIntegerOps())) {
430 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP16, RC: &ARM::MQPRRegClass, Op0);
431 }
432 if ((Subtarget->hasNEON())) {
433 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP16q, RC: &ARM::QPRRegClass, Op0);
434 }
435 return 0;
436}
437
438unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v2i32_r(unsigned Op0) {
439 if ((!Subtarget->hasSlowVDUP32()) && (Subtarget->hasNEON())) {
440 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP32d, RC: &ARM::DPRRegClass, Op0);
441 }
442 return 0;
443}
444
445unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i32_r(unsigned Op0) {
446 if ((Subtarget->hasMVEIntegerOps())) {
447 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP32, RC: &ARM::MQPRRegClass, Op0);
448 }
449 if ((Subtarget->hasNEON())) {
450 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP32q, RC: &ARM::QPRRegClass, Op0);
451 }
452 return 0;
453}
454
455unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8f16_r(unsigned Op0) {
456 if ((Subtarget->hasMVEIntegerOps())) {
457 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP16, RC: &ARM::MQPRRegClass, Op0);
458 }
459 return 0;
460}
461
462unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4f32_r(unsigned Op0) {
463 if ((Subtarget->hasMVEIntegerOps())) {
464 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP32, RC: &ARM::MQPRRegClass, Op0);
465 }
466 return 0;
467}
468
469unsigned fastEmit_ARMISD_VDUP_MVT_i32_r(MVT RetVT, unsigned Op0) {
470switch (RetVT.SimpleTy) {
471 case MVT::v8i8: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i8_r(Op0);
472 case MVT::v16i8: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v16i8_r(Op0);
473 case MVT::v4i16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i16_r(Op0);
474 case MVT::v8i16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i16_r(Op0);
475 case MVT::v2i32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v2i32_r(Op0);
476 case MVT::v4i32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i32_r(Op0);
477 case MVT::v8f16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8f16_r(Op0);
478 case MVT::v4f32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4f32_r(Op0);
479 default: return 0;
480}
481}
482
483unsigned fastEmit_ARMISD_VDUP_r(MVT VT, MVT RetVT, unsigned Op0) {
484 switch (VT.SimpleTy) {
485 case MVT::i32: return fastEmit_ARMISD_VDUP_MVT_i32_r(RetVT, Op0);
486 default: return 0;
487 }
488}
489
490// FastEmit functions for ARMISD::VMOVSR.
491
492unsigned fastEmit_ARMISD_VMOVSR_MVT_i32_r(MVT RetVT, unsigned Op0) {
493 if (RetVT.SimpleTy != MVT::f32)
494 return 0;
495 if ((Subtarget->hasFPRegs()) && (Subtarget->preferVMOVSR() ||!Subtarget->useNEONForSinglePrecisionFP())) {
496 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVSR, RC: &ARM::SPRRegClass, Op0);
497 }
498 return 0;
499}
500
501unsigned fastEmit_ARMISD_VMOVSR_r(MVT VT, MVT RetVT, unsigned Op0) {
502 switch (VT.SimpleTy) {
503 case MVT::i32: return fastEmit_ARMISD_VMOVSR_MVT_i32_r(RetVT, Op0);
504 default: return 0;
505 }
506}
507
508// FastEmit functions for ARMISD::VMOVhr.
509
510unsigned fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_bf16_r(unsigned Op0) {
511 if ((Subtarget->hasFPRegs16())) {
512 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVHR, RC: &ARM::HPRRegClass, Op0);
513 }
514 return 0;
515}
516
517unsigned fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_f16_r(unsigned Op0) {
518 if ((Subtarget->hasFPRegs16())) {
519 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVHR, RC: &ARM::HPRRegClass, Op0);
520 }
521 return 0;
522}
523
524unsigned fastEmit_ARMISD_VMOVhr_MVT_i32_r(MVT RetVT, unsigned Op0) {
525switch (RetVT.SimpleTy) {
526 case MVT::bf16: return fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_bf16_r(Op0);
527 case MVT::f16: return fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_f16_r(Op0);
528 default: return 0;
529}
530}
531
532unsigned fastEmit_ARMISD_VMOVhr_r(MVT VT, MVT RetVT, unsigned Op0) {
533 switch (VT.SimpleTy) {
534 case MVT::i32: return fastEmit_ARMISD_VMOVhr_MVT_i32_r(RetVT, Op0);
535 default: return 0;
536 }
537}
538
539// FastEmit functions for ARMISD::VMOVrh.
540
541unsigned fastEmit_ARMISD_VMOVrh_MVT_bf16_r(MVT RetVT, unsigned Op0) {
542 if (RetVT.SimpleTy != MVT::i32)
543 return 0;
544 if ((Subtarget->hasFPRegs16())) {
545 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVRH, RC: &ARM::rGPRRegClass, Op0);
546 }
547 return 0;
548}
549
550unsigned fastEmit_ARMISD_VMOVrh_MVT_f16_r(MVT RetVT, unsigned Op0) {
551 if (RetVT.SimpleTy != MVT::i32)
552 return 0;
553 if ((Subtarget->hasFPRegs16())) {
554 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVRH, RC: &ARM::rGPRRegClass, Op0);
555 }
556 return 0;
557}
558
559unsigned fastEmit_ARMISD_VMOVrh_r(MVT VT, MVT RetVT, unsigned Op0) {
560 switch (VT.SimpleTy) {
561 case MVT::bf16: return fastEmit_ARMISD_VMOVrh_MVT_bf16_r(RetVT, Op0);
562 case MVT::f16: return fastEmit_ARMISD_VMOVrh_MVT_f16_r(RetVT, Op0);
563 default: return 0;
564 }
565}
566
567// FastEmit functions for ARMISD::VREV16.
568
569unsigned fastEmit_ARMISD_VREV16_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
570 if (RetVT.SimpleTy != MVT::v8i8)
571 return 0;
572 if ((Subtarget->hasNEON())) {
573 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
574 }
575 return 0;
576}
577
578unsigned fastEmit_ARMISD_VREV16_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
579 if (RetVT.SimpleTy != MVT::v16i8)
580 return 0;
581 if ((Subtarget->hasMVEIntegerOps())) {
582 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
583 }
584 if ((Subtarget->hasNEON())) {
585 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
586 }
587 return 0;
588}
589
590unsigned fastEmit_ARMISD_VREV16_r(MVT VT, MVT RetVT, unsigned Op0) {
591 switch (VT.SimpleTy) {
592 case MVT::v8i8: return fastEmit_ARMISD_VREV16_MVT_v8i8_r(RetVT, Op0);
593 case MVT::v16i8: return fastEmit_ARMISD_VREV16_MVT_v16i8_r(RetVT, Op0);
594 default: return 0;
595 }
596}
597
598// FastEmit functions for ARMISD::VREV32.
599
600unsigned fastEmit_ARMISD_VREV32_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
601 if (RetVT.SimpleTy != MVT::v8i8)
602 return 0;
603 if ((Subtarget->hasNEON())) {
604 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
605 }
606 return 0;
607}
608
609unsigned fastEmit_ARMISD_VREV32_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
610 if (RetVT.SimpleTy != MVT::v16i8)
611 return 0;
612 if ((Subtarget->hasMVEIntegerOps())) {
613 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
614 }
615 if ((Subtarget->hasNEON())) {
616 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
617 }
618 return 0;
619}
620
621unsigned fastEmit_ARMISD_VREV32_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
622 if (RetVT.SimpleTy != MVT::v4i16)
623 return 0;
624 if ((Subtarget->hasNEON())) {
625 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
626 }
627 return 0;
628}
629
630unsigned fastEmit_ARMISD_VREV32_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
631 if (RetVT.SimpleTy != MVT::v8i16)
632 return 0;
633 if ((Subtarget->hasMVEIntegerOps())) {
634 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
635 }
636 if ((Subtarget->hasNEON())) {
637 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
638 }
639 return 0;
640}
641
642unsigned fastEmit_ARMISD_VREV32_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
643 if (RetVT.SimpleTy != MVT::v4f16)
644 return 0;
645 if ((Subtarget->hasNEON())) {
646 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
647 }
648 return 0;
649}
650
651unsigned fastEmit_ARMISD_VREV32_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
652 if (RetVT.SimpleTy != MVT::v8f16)
653 return 0;
654 if ((Subtarget->hasMVEIntegerOps())) {
655 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
656 }
657 if ((Subtarget->hasNEON())) {
658 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
659 }
660 return 0;
661}
662
663unsigned fastEmit_ARMISD_VREV32_MVT_v4bf16_r(MVT RetVT, unsigned Op0) {
664 if (RetVT.SimpleTy != MVT::v4bf16)
665 return 0;
666 if ((Subtarget->hasNEON())) {
667 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
668 }
669 return 0;
670}
671
672unsigned fastEmit_ARMISD_VREV32_MVT_v8bf16_r(MVT RetVT, unsigned Op0) {
673 if (RetVT.SimpleTy != MVT::v8bf16)
674 return 0;
675 if ((Subtarget->hasNEON())) {
676 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
677 }
678 return 0;
679}
680
681unsigned fastEmit_ARMISD_VREV32_r(MVT VT, MVT RetVT, unsigned Op0) {
682 switch (VT.SimpleTy) {
683 case MVT::v8i8: return fastEmit_ARMISD_VREV32_MVT_v8i8_r(RetVT, Op0);
684 case MVT::v16i8: return fastEmit_ARMISD_VREV32_MVT_v16i8_r(RetVT, Op0);
685 case MVT::v4i16: return fastEmit_ARMISD_VREV32_MVT_v4i16_r(RetVT, Op0);
686 case MVT::v8i16: return fastEmit_ARMISD_VREV32_MVT_v8i16_r(RetVT, Op0);
687 case MVT::v4f16: return fastEmit_ARMISD_VREV32_MVT_v4f16_r(RetVT, Op0);
688 case MVT::v8f16: return fastEmit_ARMISD_VREV32_MVT_v8f16_r(RetVT, Op0);
689 case MVT::v4bf16: return fastEmit_ARMISD_VREV32_MVT_v4bf16_r(RetVT, Op0);
690 case MVT::v8bf16: return fastEmit_ARMISD_VREV32_MVT_v8bf16_r(RetVT, Op0);
691 default: return 0;
692 }
693}
694
695// FastEmit functions for ARMISD::VREV64.
696
697unsigned fastEmit_ARMISD_VREV64_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
698 if (RetVT.SimpleTy != MVT::v8i8)
699 return 0;
700 if ((Subtarget->hasNEON())) {
701 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
702 }
703 return 0;
704}
705
706unsigned fastEmit_ARMISD_VREV64_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
707 if (RetVT.SimpleTy != MVT::v16i8)
708 return 0;
709 if ((Subtarget->hasMVEIntegerOps())) {
710 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
711 }
712 if ((Subtarget->hasNEON())) {
713 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
714 }
715 return 0;
716}
717
718unsigned fastEmit_ARMISD_VREV64_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
719 if (RetVT.SimpleTy != MVT::v4i16)
720 return 0;
721 if ((Subtarget->hasNEON())) {
722 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
723 }
724 return 0;
725}
726
727unsigned fastEmit_ARMISD_VREV64_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
728 if (RetVT.SimpleTy != MVT::v8i16)
729 return 0;
730 if ((Subtarget->hasMVEIntegerOps())) {
731 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
732 }
733 if ((Subtarget->hasNEON())) {
734 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
735 }
736 return 0;
737}
738
739unsigned fastEmit_ARMISD_VREV64_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
740 if (RetVT.SimpleTy != MVT::v2i32)
741 return 0;
742 if ((Subtarget->hasNEON())) {
743 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
744 }
745 return 0;
746}
747
748unsigned fastEmit_ARMISD_VREV64_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
749 if (RetVT.SimpleTy != MVT::v4i32)
750 return 0;
751 if ((Subtarget->hasMVEIntegerOps())) {
752 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
753 }
754 if ((Subtarget->hasNEON())) {
755 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
756 }
757 return 0;
758}
759
760unsigned fastEmit_ARMISD_VREV64_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
761 if (RetVT.SimpleTy != MVT::v4f16)
762 return 0;
763 if ((Subtarget->hasNEON())) {
764 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
765 }
766 return 0;
767}
768
769unsigned fastEmit_ARMISD_VREV64_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
770 if (RetVT.SimpleTy != MVT::v8f16)
771 return 0;
772 if ((Subtarget->hasMVEIntegerOps())) {
773 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
774 }
775 if ((Subtarget->hasNEON())) {
776 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
777 }
778 return 0;
779}
780
781unsigned fastEmit_ARMISD_VREV64_MVT_v4bf16_r(MVT RetVT, unsigned Op0) {
782 if (RetVT.SimpleTy != MVT::v4bf16)
783 return 0;
784 if ((Subtarget->hasNEON())) {
785 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
786 }
787 return 0;
788}
789
790unsigned fastEmit_ARMISD_VREV64_MVT_v8bf16_r(MVT RetVT, unsigned Op0) {
791 if (RetVT.SimpleTy != MVT::v8bf16)
792 return 0;
793 if ((Subtarget->hasNEON())) {
794 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
795 }
796 return 0;
797}
798
799unsigned fastEmit_ARMISD_VREV64_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
800 if (RetVT.SimpleTy != MVT::v2f32)
801 return 0;
802 if ((Subtarget->hasNEON())) {
803 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
804 }
805 return 0;
806}
807
808unsigned fastEmit_ARMISD_VREV64_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
809 if (RetVT.SimpleTy != MVT::v4f32)
810 return 0;
811 if ((Subtarget->hasMVEIntegerOps())) {
812 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
813 }
814 if ((Subtarget->hasNEON())) {
815 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
816 }
817 return 0;
818}
819
820unsigned fastEmit_ARMISD_VREV64_r(MVT VT, MVT RetVT, unsigned Op0) {
821 switch (VT.SimpleTy) {
822 case MVT::v8i8: return fastEmit_ARMISD_VREV64_MVT_v8i8_r(RetVT, Op0);
823 case MVT::v16i8: return fastEmit_ARMISD_VREV64_MVT_v16i8_r(RetVT, Op0);
824 case MVT::v4i16: return fastEmit_ARMISD_VREV64_MVT_v4i16_r(RetVT, Op0);
825 case MVT::v8i16: return fastEmit_ARMISD_VREV64_MVT_v8i16_r(RetVT, Op0);
826 case MVT::v2i32: return fastEmit_ARMISD_VREV64_MVT_v2i32_r(RetVT, Op0);
827 case MVT::v4i32: return fastEmit_ARMISD_VREV64_MVT_v4i32_r(RetVT, Op0);
828 case MVT::v4f16: return fastEmit_ARMISD_VREV64_MVT_v4f16_r(RetVT, Op0);
829 case MVT::v8f16: return fastEmit_ARMISD_VREV64_MVT_v8f16_r(RetVT, Op0);
830 case MVT::v4bf16: return fastEmit_ARMISD_VREV64_MVT_v4bf16_r(RetVT, Op0);
831 case MVT::v8bf16: return fastEmit_ARMISD_VREV64_MVT_v8bf16_r(RetVT, Op0);
832 case MVT::v2f32: return fastEmit_ARMISD_VREV64_MVT_v2f32_r(RetVT, Op0);
833 case MVT::v4f32: return fastEmit_ARMISD_VREV64_MVT_v4f32_r(RetVT, Op0);
834 default: return 0;
835 }
836}
837
838// FastEmit functions for ARMISD::WIN__DBZCHK.
839
840unsigned fastEmit_ARMISD_WIN__DBZCHK_MVT_i32_r(MVT RetVT, unsigned Op0) {
841 if (RetVT.SimpleTy != MVT::isVoid)
842 return 0;
843 return fastEmitInst_r(MachineInstOpcode: ARM::WIN__DBZCHK, RC: &ARM::tGPRRegClass, Op0);
844}
845
846unsigned fastEmit_ARMISD_WIN__DBZCHK_r(MVT VT, MVT RetVT, unsigned Op0) {
847 switch (VT.SimpleTy) {
848 case MVT::i32: return fastEmit_ARMISD_WIN__DBZCHK_MVT_i32_r(RetVT, Op0);
849 default: return 0;
850 }
851}
852
853// FastEmit functions for ARMISD::tSECALL.
854
855unsigned fastEmit_ARMISD_tSECALL_MVT_i32_r(MVT RetVT, unsigned Op0) {
856 if (RetVT.SimpleTy != MVT::isVoid)
857 return 0;
858 if ((Subtarget->has8MSecExt()) && (Subtarget->isThumb())) {
859 return fastEmitInst_r(MachineInstOpcode: ARM::tBLXNS_CALL, RC: &ARM::GPRnopcRegClass, Op0);
860 }
861 return 0;
862}
863
864unsigned fastEmit_ARMISD_tSECALL_r(MVT VT, MVT RetVT, unsigned Op0) {
865 switch (VT.SimpleTy) {
866 case MVT::i32: return fastEmit_ARMISD_tSECALL_MVT_i32_r(RetVT, Op0);
867 default: return 0;
868 }
869}
870
871// FastEmit functions for ISD::ABS.
872
873unsigned fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
874 if (RetVT.SimpleTy != MVT::v8i8)
875 return 0;
876 if ((Subtarget->hasNEON())) {
877 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv8i8, RC: &ARM::DPRRegClass, Op0);
878 }
879 return 0;
880}
881
882unsigned fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
883 if (RetVT.SimpleTy != MVT::v16i8)
884 return 0;
885 if ((Subtarget->hasMVEIntegerOps())) {
886 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSs8, RC: &ARM::MQPRRegClass, Op0);
887 }
888 if ((Subtarget->hasNEON())) {
889 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv16i8, RC: &ARM::QPRRegClass, Op0);
890 }
891 return 0;
892}
893
894unsigned fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
895 if (RetVT.SimpleTy != MVT::v4i16)
896 return 0;
897 if ((Subtarget->hasNEON())) {
898 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv4i16, RC: &ARM::DPRRegClass, Op0);
899 }
900 return 0;
901}
902
903unsigned fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
904 if (RetVT.SimpleTy != MVT::v8i16)
905 return 0;
906 if ((Subtarget->hasMVEIntegerOps())) {
907 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSs16, RC: &ARM::MQPRRegClass, Op0);
908 }
909 if ((Subtarget->hasNEON())) {
910 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv8i16, RC: &ARM::QPRRegClass, Op0);
911 }
912 return 0;
913}
914
915unsigned fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
916 if (RetVT.SimpleTy != MVT::v2i32)
917 return 0;
918 if ((Subtarget->hasNEON())) {
919 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv2i32, RC: &ARM::DPRRegClass, Op0);
920 }
921 return 0;
922}
923
924unsigned fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
925 if (RetVT.SimpleTy != MVT::v4i32)
926 return 0;
927 if ((Subtarget->hasMVEIntegerOps())) {
928 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSs32, RC: &ARM::MQPRRegClass, Op0);
929 }
930 if ((Subtarget->hasNEON())) {
931 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv4i32, RC: &ARM::QPRRegClass, Op0);
932 }
933 return 0;
934}
935
936unsigned fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, unsigned Op0) {
937 switch (VT.SimpleTy) {
938 case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0);
939 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
940 case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0);
941 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
942 case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0);
943 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
944 default: return 0;
945 }
946}
947
948// FastEmit functions for ISD::ANY_EXTEND.
949
950unsigned fastEmit_ISD_ANY_EXTEND_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
951 if (RetVT.SimpleTy != MVT::v8i16)
952 return 0;
953 if ((Subtarget->hasNEON())) {
954 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv8i16, RC: &ARM::QPRRegClass, Op0);
955 }
956 return 0;
957}
958
959unsigned fastEmit_ISD_ANY_EXTEND_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
960 if (RetVT.SimpleTy != MVT::v4i32)
961 return 0;
962 if ((Subtarget->hasNEON())) {
963 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv4i32, RC: &ARM::QPRRegClass, Op0);
964 }
965 return 0;
966}
967
968unsigned fastEmit_ISD_ANY_EXTEND_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
969 if (RetVT.SimpleTy != MVT::v2i64)
970 return 0;
971 if ((Subtarget->hasNEON())) {
972 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv2i64, RC: &ARM::QPRRegClass, Op0);
973 }
974 return 0;
975}
976
977unsigned fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) {
978 switch (VT.SimpleTy) {
979 case MVT::v8i8: return fastEmit_ISD_ANY_EXTEND_MVT_v8i8_r(RetVT, Op0);
980 case MVT::v4i16: return fastEmit_ISD_ANY_EXTEND_MVT_v4i16_r(RetVT, Op0);
981 case MVT::v2i32: return fastEmit_ISD_ANY_EXTEND_MVT_v2i32_r(RetVT, Op0);
982 default: return 0;
983 }
984}
985
986// FastEmit functions for ISD::BITCAST.
987
988unsigned fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, unsigned Op0) {
989 if (RetVT.SimpleTy != MVT::f32)
990 return 0;
991 if ((Subtarget->hasFPRegs()) && (Subtarget->preferVMOVSR() ||!Subtarget->useNEONForSinglePrecisionFP())) {
992 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVSR, RC: &ARM::SPRRegClass, Op0);
993 }
994 return 0;
995}
996
997unsigned fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, unsigned Op0) {
998 if (RetVT.SimpleTy != MVT::i32)
999 return 0;
1000 if ((Subtarget->hasFPRegs())) {
1001 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVRS, RC: &ARM::GPRRegClass, Op0);
1002 }
1003 return 0;
1004}
1005
1006unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(unsigned Op0) {
1007 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1008 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
1009 }
1010 return 0;
1011}
1012
1013unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(unsigned Op0) {
1014 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1015 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1016 }
1017 return 0;
1018}
1019
1020unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(unsigned Op0) {
1021 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1022 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1023 }
1024 return 0;
1025}
1026
1027unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(unsigned Op0) {
1028 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1029 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1030 }
1031 return 0;
1032}
1033
1034unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(unsigned Op0) {
1035 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1036 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1037 }
1038 return 0;
1039}
1040
1041unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(unsigned Op0) {
1042 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1043 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1044 }
1045 return 0;
1046}
1047
1048unsigned fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, unsigned Op0) {
1049switch (RetVT.SimpleTy) {
1050 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0);
1051 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0);
1052 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0);
1053 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0);
1054 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Op0);
1055 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0);
1056 default: return 0;
1057}
1058}
1059
1060unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(unsigned Op0) {
1061 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1062 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
1063 }
1064 return 0;
1065}
1066
1067unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(unsigned Op0) {
1068 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1069 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1070 }
1071 return 0;
1072}
1073
1074unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(unsigned Op0) {
1075 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1076 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1077 }
1078 return 0;
1079}
1080
1081unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(unsigned Op0) {
1082 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1083 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
1084 }
1085 return 0;
1086}
1087
1088unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(unsigned Op0) {
1089 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1090 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1091 }
1092 return 0;
1093}
1094
1095unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(unsigned Op0) {
1096 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1097 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1098 }
1099 return 0;
1100}
1101
1102unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(unsigned Op0) {
1103 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1104 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1105 }
1106 return 0;
1107}
1108
1109unsigned fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
1110switch (RetVT.SimpleTy) {
1111 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0);
1112 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0);
1113 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0);
1114 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0);
1115 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0);
1116 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Op0);
1117 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0);
1118 default: return 0;
1119}
1120}
1121
1122unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(unsigned Op0) {
1123 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1124 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1125 }
1126 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1127 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1128 }
1129 return 0;
1130}
1131
1132unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(unsigned Op0) {
1133 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1134 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1135 }
1136 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1137 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1138 }
1139 return 0;
1140}
1141
1142unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(unsigned Op0) {
1143 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1144 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1145 }
1146 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1147 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1148 }
1149 return 0;
1150}
1151
1152unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(unsigned Op0) {
1153 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1154 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1155 }
1156 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1157 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1158 }
1159 return 0;
1160}
1161
1162unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(unsigned Op0) {
1163 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1164 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1165 }
1166 return 0;
1167}
1168
1169unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(unsigned Op0) {
1170 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1171 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1172 }
1173 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1174 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1175 }
1176 return 0;
1177}
1178
1179unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(unsigned Op0) {
1180 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1181 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1182 }
1183 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1184 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1185 }
1186 return 0;
1187}
1188
1189unsigned fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
1190switch (RetVT.SimpleTy) {
1191 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0);
1192 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0);
1193 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0);
1194 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0);
1195 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Op0);
1196 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0);
1197 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0);
1198 default: return 0;
1199}
1200}
1201
1202unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(unsigned Op0) {
1203 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1204 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1205 }
1206 return 0;
1207}
1208
1209unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(unsigned Op0) {
1210 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1211 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1212 }
1213 return 0;
1214}
1215
1216unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(unsigned Op0) {
1217 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1218 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1219 }
1220 return 0;
1221}
1222
1223unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(unsigned Op0) {
1224 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1225 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1226 }
1227 return 0;
1228}
1229
1230unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(unsigned Op0) {
1231 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1232 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1233 }
1234 return 0;
1235}
1236
1237unsigned fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
1238switch (RetVT.SimpleTy) {
1239 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0);
1240 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0);
1241 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0);
1242 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0);
1243 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0);
1244 default: return 0;
1245}
1246}
1247
1248unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(unsigned Op0) {
1249 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1250 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1251 }
1252 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1253 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1254 }
1255 return 0;
1256}
1257
1258unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(unsigned Op0) {
1259 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1260 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1261 }
1262 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1263 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1264 }
1265 return 0;
1266}
1267
1268unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(unsigned Op0) {
1269 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1270 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1271 }
1272 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1273 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1274 }
1275 return 0;
1276}
1277
1278unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(unsigned Op0) {
1279 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1280 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1281 }
1282 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1283 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1284 }
1285 return 0;
1286}
1287
1288unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(unsigned Op0) {
1289 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1290 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1291 }
1292 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1293 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1294 }
1295 return 0;
1296}
1297
1298unsigned fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
1299switch (RetVT.SimpleTy) {
1300 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0);
1301 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0);
1302 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0);
1303 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0);
1304 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0);
1305 default: return 0;
1306}
1307}
1308
1309unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(unsigned Op0) {
1310 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1311 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1312 }
1313 return 0;
1314}
1315
1316unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(unsigned Op0) {
1317 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1318 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1319 }
1320 return 0;
1321}
1322
1323unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(unsigned Op0) {
1324 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1325 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1326 }
1327 return 0;
1328}
1329
1330unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(unsigned Op0) {
1331 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1332 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1333 }
1334 return 0;
1335}
1336
1337unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(unsigned Op0) {
1338 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1339 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1340 }
1341 return 0;
1342}
1343
1344unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(unsigned Op0) {
1345 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1346 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1347 }
1348 return 0;
1349}
1350
1351unsigned fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
1352switch (RetVT.SimpleTy) {
1353 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0);
1354 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0);
1355 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0);
1356 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0);
1357 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0);
1358 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Op0);
1359 default: return 0;
1360}
1361}
1362
1363unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(unsigned Op0) {
1364 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1365 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1366 }
1367 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1368 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1369 }
1370 return 0;
1371}
1372
1373unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(unsigned Op0) {
1374 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1375 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1376 }
1377 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1378 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1379 }
1380 return 0;
1381}
1382
1383unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(unsigned Op0) {
1384 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1385 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1386 }
1387 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1388 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1389 }
1390 return 0;
1391}
1392
1393unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(unsigned Op0) {
1394 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1395 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1396 }
1397 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1398 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1399 }
1400 return 0;
1401}
1402
1403unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(unsigned Op0) {
1404 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1405 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1406 }
1407 return 0;
1408}
1409
1410unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(unsigned Op0) {
1411 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1412 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1413 }
1414 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1415 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1416 }
1417 return 0;
1418}
1419
1420unsigned fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
1421switch (RetVT.SimpleTy) {
1422 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0);
1423 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0);
1424 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0);
1425 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0);
1426 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Op0);
1427 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0);
1428 default: return 0;
1429}
1430}
1431
1432unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(unsigned Op0) {
1433 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1434 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
1435 }
1436 return 0;
1437}
1438
1439unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(unsigned Op0) {
1440 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1441 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1442 }
1443 return 0;
1444}
1445
1446unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(unsigned Op0) {
1447 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1448 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1449 }
1450 return 0;
1451}
1452
1453unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(unsigned Op0) {
1454 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1455 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1456 }
1457 return 0;
1458}
1459
1460unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(unsigned Op0) {
1461 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1462 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1463 }
1464 return 0;
1465}
1466
1467unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(unsigned Op0) {
1468 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1469 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1470 }
1471 return 0;
1472}
1473
1474unsigned fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, unsigned Op0) {
1475switch (RetVT.SimpleTy) {
1476 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0);
1477 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0);
1478 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0);
1479 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0);
1480 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Op0);
1481 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0);
1482 default: return 0;
1483}
1484}
1485
1486unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(unsigned Op0) {
1487 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1488 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1489 }
1490 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1491 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1492 }
1493 return 0;
1494}
1495
1496unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(unsigned Op0) {
1497 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1498 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1499 }
1500 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1501 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1502 }
1503 return 0;
1504}
1505
1506unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(unsigned Op0) {
1507 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1508 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1509 }
1510 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1511 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1512 }
1513 return 0;
1514}
1515
1516unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(unsigned Op0) {
1517 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1518 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1519 }
1520 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1521 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1522 }
1523 return 0;
1524}
1525
1526unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(unsigned Op0) {
1527 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1528 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1529 }
1530 return 0;
1531}
1532
1533unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(unsigned Op0) {
1534 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1535 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1536 }
1537 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1538 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1539 }
1540 return 0;
1541}
1542
1543unsigned fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
1544switch (RetVT.SimpleTy) {
1545 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0);
1546 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0);
1547 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0);
1548 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0);
1549 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Op0);
1550 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0);
1551 default: return 0;
1552}
1553}
1554
1555unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(unsigned Op0) {
1556 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1557 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1558 }
1559 return 0;
1560}
1561
1562unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(unsigned Op0) {
1563 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1564 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1565 }
1566 return 0;
1567}
1568
1569unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(unsigned Op0) {
1570 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1571 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1572 }
1573 return 0;
1574}
1575
1576unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(unsigned Op0) {
1577 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1578 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1579 }
1580 return 0;
1581}
1582
1583unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(unsigned Op0) {
1584 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1585 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1586 }
1587 return 0;
1588}
1589
1590unsigned fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
1591switch (RetVT.SimpleTy) {
1592 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0);
1593 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0);
1594 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0);
1595 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0);
1596 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0);
1597 default: return 0;
1598}
1599}
1600
1601unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(unsigned Op0) {
1602 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1603 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1604 }
1605 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1606 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1607 }
1608 return 0;
1609}
1610
1611unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(unsigned Op0) {
1612 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1613 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1614 }
1615 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1616 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1617 }
1618 return 0;
1619}
1620
1621unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(unsigned Op0) {
1622 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1623 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1624 }
1625 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1626 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1627 }
1628 return 0;
1629}
1630
1631unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(unsigned Op0) {
1632 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1633 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1634 }
1635 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1636 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1637 }
1638 return 0;
1639}
1640
1641unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(unsigned Op0) {
1642 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1643 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1644 }
1645 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1646 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1647 }
1648 return 0;
1649}
1650
1651unsigned fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
1652switch (RetVT.SimpleTy) {
1653 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0);
1654 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0);
1655 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0);
1656 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0);
1657 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0);
1658 default: return 0;
1659}
1660}
1661
1662unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(unsigned Op0) {
1663 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1664 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1665 }
1666 return 0;
1667}
1668
1669unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(unsigned Op0) {
1670 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1671 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1672 }
1673 return 0;
1674}
1675
1676unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(unsigned Op0) {
1677 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1678 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1679 }
1680 return 0;
1681}
1682
1683unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(unsigned Op0) {
1684 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1685 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1686 }
1687 return 0;
1688}
1689
1690unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(unsigned Op0) {
1691 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1692 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1693 }
1694 return 0;
1695}
1696
1697unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_r(MVT RetVT, unsigned Op0) {
1698switch (RetVT.SimpleTy) {
1699 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Op0);
1700 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Op0);
1701 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Op0);
1702 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Op0);
1703 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Op0);
1704 default: return 0;
1705}
1706}
1707
1708unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(unsigned Op0) {
1709 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1710 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1711 }
1712 return 0;
1713}
1714
1715unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(unsigned Op0) {
1716 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1717 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1718 }
1719 return 0;
1720}
1721
1722unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(unsigned Op0) {
1723 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1724 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1725 }
1726 return 0;
1727}
1728
1729unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(unsigned Op0) {
1730 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1731 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1732 }
1733 return 0;
1734}
1735
1736unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(unsigned Op0) {
1737 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1738 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1739 }
1740 return 0;
1741}
1742
1743unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_r(MVT RetVT, unsigned Op0) {
1744switch (RetVT.SimpleTy) {
1745 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Op0);
1746 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Op0);
1747 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Op0);
1748 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Op0);
1749 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Op0);
1750 default: return 0;
1751}
1752}
1753
1754unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(unsigned Op0) {
1755 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1756 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1757 }
1758 return 0;
1759}
1760
1761unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(unsigned Op0) {
1762 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1763 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1764 }
1765 return 0;
1766}
1767
1768unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(unsigned Op0) {
1769 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1770 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1771 }
1772 return 0;
1773}
1774
1775unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(unsigned Op0) {
1776 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1777 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1778 }
1779 return 0;
1780}
1781
1782unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(unsigned Op0) {
1783 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1784 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1785 }
1786 return 0;
1787}
1788
1789unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(unsigned Op0) {
1790 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1791 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1792 }
1793 return 0;
1794}
1795
1796unsigned fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
1797switch (RetVT.SimpleTy) {
1798 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0);
1799 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0);
1800 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0);
1801 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0);
1802 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0);
1803 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Op0);
1804 default: return 0;
1805}
1806}
1807
1808unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(unsigned Op0) {
1809 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1810 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1811 }
1812 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1813 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1814 }
1815 return 0;
1816}
1817
1818unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(unsigned Op0) {
1819 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1820 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1821 }
1822 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1823 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1824 }
1825 return 0;
1826}
1827
1828unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(unsigned Op0) {
1829 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1830 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1831 }
1832 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1833 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1834 }
1835 return 0;
1836}
1837
1838unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(unsigned Op0) {
1839 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1840 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1841 }
1842 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1843 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1844 }
1845 return 0;
1846}
1847
1848unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(unsigned Op0) {
1849 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1850 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1851 }
1852 return 0;
1853}
1854
1855unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(unsigned Op0) {
1856 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1857 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1858 }
1859 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1860 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1861 }
1862 return 0;
1863}
1864
1865unsigned fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
1866switch (RetVT.SimpleTy) {
1867 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0);
1868 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0);
1869 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0);
1870 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0);
1871 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Op0);
1872 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0);
1873 default: return 0;
1874}
1875}
1876
1877unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(unsigned Op0) {
1878 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1879 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1880 }
1881 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1882 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1883 }
1884 return 0;
1885}
1886
1887unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(unsigned Op0) {
1888 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1889 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1890 }
1891 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1892 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1893 }
1894 return 0;
1895}
1896
1897unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(unsigned Op0) {
1898 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1899 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1900 }
1901 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1902 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1903 }
1904 return 0;
1905}
1906
1907unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(unsigned Op0) {
1908 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1909 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1910 }
1911 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1912 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1913 }
1914 return 0;
1915}
1916
1917unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(unsigned Op0) {
1918 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1919 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1920 }
1921 return 0;
1922}
1923
1924unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(unsigned Op0) {
1925 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1926 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1927 }
1928 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1929 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1930 }
1931 return 0;
1932}
1933
1934unsigned fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
1935switch (RetVT.SimpleTy) {
1936 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0);
1937 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0);
1938 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0);
1939 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0);
1940 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Op0);
1941 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0);
1942 default: return 0;
1943}
1944}
1945
1946unsigned fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, unsigned Op0) {
1947 switch (VT.SimpleTy) {
1948 case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0);
1949 case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0);
1950 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
1951 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0);
1952 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0);
1953 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0);
1954 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0);
1955 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0);
1956 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0);
1957 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0);
1958 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0);
1959 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0);
1960 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0);
1961 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v4bf16_r(RetVT, Op0);
1962 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v8bf16_r(RetVT, Op0);
1963 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0);
1964 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0);
1965 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0);
1966 default: return 0;
1967 }
1968}
1969
1970// FastEmit functions for ISD::BITREVERSE.
1971
1972unsigned fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, unsigned Op0) {
1973 if (RetVT.SimpleTy != MVT::i32)
1974 return 0;
1975 if ((Subtarget->isThumb2())) {
1976 return fastEmitInst_r(MachineInstOpcode: ARM::t2RBIT, RC: &ARM::rGPRRegClass, Op0);
1977 }
1978 if ((Subtarget->hasV6T2Ops()) && (!Subtarget->isThumb())) {
1979 return fastEmitInst_r(MachineInstOpcode: ARM::RBIT, RC: &ARM::GPRRegClass, Op0);
1980 }
1981 return 0;
1982}
1983
1984unsigned fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, unsigned Op0) {
1985 switch (VT.SimpleTy) {
1986 case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0);
1987 default: return 0;
1988 }
1989}
1990
1991// FastEmit functions for ISD::BRIND.
1992
1993unsigned fastEmit_ISD_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0) {
1994 if (RetVT.SimpleTy != MVT::isVoid)
1995 return 0;
1996 if ((Subtarget->isThumb())) {
1997 return fastEmitInst_r(MachineInstOpcode: ARM::tBRIND, RC: &ARM::GPRRegClass, Op0);
1998 }
1999 if ((!Subtarget->isThumb()) && (!Subtarget->hasV4TOps())) {
2000 return fastEmitInst_r(MachineInstOpcode: ARM::MOVPCRX, RC: &ARM::GPRRegClass, Op0);
2001 }
2002 if ((Subtarget->hasV4TOps()) && (!Subtarget->isThumb())) {
2003 return fastEmitInst_r(MachineInstOpcode: ARM::BX, RC: &ARM::GPRRegClass, Op0);
2004 }
2005 return 0;
2006}
2007
2008unsigned fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, unsigned Op0) {
2009 switch (VT.SimpleTy) {
2010 case MVT::i32: return fastEmit_ISD_BRIND_MVT_i32_r(RetVT, Op0);
2011 default: return 0;
2012 }
2013}
2014
2015// FastEmit functions for ISD::BSWAP.
2016
2017unsigned fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, unsigned Op0) {
2018 if (RetVT.SimpleTy != MVT::i32)
2019 return 0;
2020 if ((Subtarget->isThumb2())) {
2021 return fastEmitInst_r(MachineInstOpcode: ARM::t2REV, RC: &ARM::rGPRRegClass, Op0);
2022 }
2023 if ((Subtarget->hasV6Ops()) && (Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
2024 return fastEmitInst_r(MachineInstOpcode: ARM::tREV, RC: &ARM::tGPRRegClass, Op0);
2025 }
2026 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
2027 return fastEmitInst_r(MachineInstOpcode: ARM::REV, RC: &ARM::GPRRegClass, Op0);
2028 }
2029 return 0;
2030}
2031
2032unsigned fastEmit_ISD_BSWAP_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
2033 if (RetVT.SimpleTy != MVT::v8i16)
2034 return 0;
2035 if ((Subtarget->hasMVEIntegerOps())) {
2036 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
2037 }
2038 return 0;
2039}
2040
2041unsigned fastEmit_ISD_BSWAP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
2042 if (RetVT.SimpleTy != MVT::v4i32)
2043 return 0;
2044 if ((Subtarget->hasMVEIntegerOps())) {
2045 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
2046 }
2047 return 0;
2048}
2049
2050unsigned fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, unsigned Op0) {
2051 switch (VT.SimpleTy) {
2052 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
2053 case MVT::v8i16: return fastEmit_ISD_BSWAP_MVT_v8i16_r(RetVT, Op0);
2054 case MVT::v4i32: return fastEmit_ISD_BSWAP_MVT_v4i32_r(RetVT, Op0);
2055 default: return 0;
2056 }
2057}
2058
2059// FastEmit functions for ISD::CTLZ.
2060
2061unsigned fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, unsigned Op0) {
2062 if (RetVT.SimpleTy != MVT::i32)
2063 return 0;
2064 if ((Subtarget->isThumb2())) {
2065 return fastEmitInst_r(MachineInstOpcode: ARM::t2CLZ, RC: &ARM::rGPRRegClass, Op0);
2066 }
2067 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb())) {
2068 return fastEmitInst_r(MachineInstOpcode: ARM::CLZ, RC: &ARM::GPRRegClass, Op0);
2069 }
2070 return 0;
2071}
2072
2073unsigned fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
2074 if (RetVT.SimpleTy != MVT::v8i8)
2075 return 0;
2076 if ((Subtarget->hasNEON())) {
2077 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv8i8, RC: &ARM::DPRRegClass, Op0);
2078 }
2079 return 0;
2080}
2081
2082unsigned fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
2083 if (RetVT.SimpleTy != MVT::v16i8)
2084 return 0;
2085 if ((Subtarget->hasMVEIntegerOps())) {
2086 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLZs8, RC: &ARM::MQPRRegClass, Op0);
2087 }
2088 if ((Subtarget->hasNEON())) {
2089 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv16i8, RC: &ARM::QPRRegClass, Op0);
2090 }
2091 return 0;
2092}
2093
2094unsigned fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
2095 if (RetVT.SimpleTy != MVT::v4i16)
2096 return 0;
2097 if ((Subtarget->hasNEON())) {
2098 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv4i16, RC: &ARM::DPRRegClass, Op0);
2099 }
2100 return 0;
2101}
2102
2103unsigned fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
2104 if (RetVT.SimpleTy != MVT::v8i16)
2105 return 0;
2106 if ((Subtarget->hasMVEIntegerOps())) {
2107 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLZs16, RC: &ARM::MQPRRegClass, Op0);
2108 }
2109 if ((Subtarget->hasNEON())) {
2110 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv8i16, RC: &ARM::QPRRegClass, Op0);
2111 }
2112 return 0;
2113}
2114
2115unsigned fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
2116 if (RetVT.SimpleTy != MVT::v2i32)
2117 return 0;
2118 if ((Subtarget->hasNEON())) {
2119 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv2i32, RC: &ARM::DPRRegClass, Op0);
2120 }
2121 return 0;
2122}
2123
2124unsigned fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
2125 if (RetVT.SimpleTy != MVT::v4i32)
2126 return 0;
2127 if ((Subtarget->hasMVEIntegerOps())) {
2128 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLZs32, RC: &ARM::MQPRRegClass, Op0);
2129 }
2130 if ((Subtarget->hasNEON())) {
2131 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv4i32, RC: &ARM::QPRRegClass, Op0);
2132 }
2133 return 0;
2134}
2135
2136unsigned fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, unsigned Op0) {
2137 switch (VT.SimpleTy) {
2138 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
2139 case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0);
2140 case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0);
2141 case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0);
2142 case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0);
2143 case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0);
2144 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
2145 default: return 0;
2146 }
2147}
2148
2149// FastEmit functions for ISD::CTPOP.
2150
2151unsigned fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
2152 if (RetVT.SimpleTy != MVT::v8i8)
2153 return 0;
2154 if ((Subtarget->hasNEON())) {
2155 return fastEmitInst_r(MachineInstOpcode: ARM::VCNTd, RC: &ARM::DPRRegClass, Op0);
2156 }
2157 return 0;
2158}
2159
2160unsigned fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
2161 if (RetVT.SimpleTy != MVT::v16i8)
2162 return 0;
2163 if ((Subtarget->hasNEON())) {
2164 return fastEmitInst_r(MachineInstOpcode: ARM::VCNTq, RC: &ARM::QPRRegClass, Op0);
2165 }
2166 return 0;
2167}
2168
2169unsigned fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, unsigned Op0) {
2170 switch (VT.SimpleTy) {
2171 case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0);
2172 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
2173 default: return 0;
2174 }
2175}
2176
2177// FastEmit functions for ISD::FABS.
2178
2179unsigned fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, unsigned Op0) {
2180 if (RetVT.SimpleTy != MVT::f16)
2181 return 0;
2182 if ((Subtarget->hasFullFP16())) {
2183 return fastEmitInst_r(MachineInstOpcode: ARM::VABSH, RC: &ARM::HPRRegClass, Op0);
2184 }
2185 return 0;
2186}
2187
2188unsigned fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, unsigned Op0) {
2189 if (RetVT.SimpleTy != MVT::f32)
2190 return 0;
2191 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
2192 return fastEmitInst_r(MachineInstOpcode: ARM::VABSS, RC: &ARM::SPRRegClass, Op0);
2193 }
2194 return 0;
2195}
2196
2197unsigned fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, unsigned Op0) {
2198 if (RetVT.SimpleTy != MVT::f64)
2199 return 0;
2200 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2201 return fastEmitInst_r(MachineInstOpcode: ARM::VABSD, RC: &ARM::DPRRegClass, Op0);
2202 }
2203 return 0;
2204}
2205
2206unsigned fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
2207 if (RetVT.SimpleTy != MVT::v4f16)
2208 return 0;
2209 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2210 return fastEmitInst_r(MachineInstOpcode: ARM::VABShd, RC: &ARM::DPRRegClass, Op0);
2211 }
2212 return 0;
2213}
2214
2215unsigned fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
2216 if (RetVT.SimpleTy != MVT::v8f16)
2217 return 0;
2218 if ((Subtarget->hasMVEIntegerOps())) {
2219 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSf16, RC: &ARM::MQPRRegClass, Op0);
2220 }
2221 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2222 return fastEmitInst_r(MachineInstOpcode: ARM::VABShq, RC: &ARM::QPRRegClass, Op0);
2223 }
2224 return 0;
2225}
2226
2227unsigned fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
2228 if (RetVT.SimpleTy != MVT::v2f32)
2229 return 0;
2230 if ((Subtarget->hasNEON())) {
2231 return fastEmitInst_r(MachineInstOpcode: ARM::VABSfd, RC: &ARM::DPRRegClass, Op0);
2232 }
2233 return 0;
2234}
2235
2236unsigned fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
2237 if (RetVT.SimpleTy != MVT::v4f32)
2238 return 0;
2239 if ((Subtarget->hasMVEIntegerOps())) {
2240 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSf32, RC: &ARM::MQPRRegClass, Op0);
2241 }
2242 if ((Subtarget->hasNEON())) {
2243 return fastEmitInst_r(MachineInstOpcode: ARM::VABSfq, RC: &ARM::QPRRegClass, Op0);
2244 }
2245 return 0;
2246}
2247
2248unsigned fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, unsigned Op0) {
2249 switch (VT.SimpleTy) {
2250 case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0);
2251 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
2252 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
2253 case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0);
2254 case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0);
2255 case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0);
2256 case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0);
2257 default: return 0;
2258 }
2259}
2260
2261// FastEmit functions for ISD::FCEIL.
2262
2263unsigned fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, unsigned Op0) {
2264 if (RetVT.SimpleTy != MVT::f16)
2265 return 0;
2266 if ((Subtarget->hasFullFP16())) {
2267 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPH, RC: &ARM::HPRRegClass, Op0);
2268 }
2269 return 0;
2270}
2271
2272unsigned fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, unsigned Op0) {
2273 if (RetVT.SimpleTy != MVT::f32)
2274 return 0;
2275 if ((Subtarget->hasFPARMv8Base())) {
2276 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPS, RC: &ARM::SPRRegClass, Op0);
2277 }
2278 return 0;
2279}
2280
2281unsigned fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, unsigned Op0) {
2282 if (RetVT.SimpleTy != MVT::f64)
2283 return 0;
2284 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2285 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPD, RC: &ARM::DPRRegClass, Op0);
2286 }
2287 return 0;
2288}
2289
2290unsigned fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
2291 if (RetVT.SimpleTy != MVT::v8f16)
2292 return 0;
2293 if ((Subtarget->hasMVEFloatOps())) {
2294 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16P, RC: &ARM::MQPRRegClass, Op0);
2295 }
2296 return 0;
2297}
2298
2299unsigned fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
2300 if (RetVT.SimpleTy != MVT::v4f32)
2301 return 0;
2302 if ((Subtarget->hasMVEFloatOps())) {
2303 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32P, RC: &ARM::MQPRRegClass, Op0);
2304 }
2305 return 0;
2306}
2307
2308unsigned fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, unsigned Op0) {
2309 switch (VT.SimpleTy) {
2310 case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0);
2311 case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0);
2312 case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0);
2313 case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0);
2314 case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0);
2315 default: return 0;
2316 }
2317}
2318
2319// FastEmit functions for ISD::FFLOOR.
2320
2321unsigned fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, unsigned Op0) {
2322 if (RetVT.SimpleTy != MVT::f16)
2323 return 0;
2324 if ((Subtarget->hasFullFP16())) {
2325 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMH, RC: &ARM::HPRRegClass, Op0);
2326 }
2327 return 0;
2328}
2329
2330unsigned fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, unsigned Op0) {
2331 if (RetVT.SimpleTy != MVT::f32)
2332 return 0;
2333 if ((Subtarget->hasFPARMv8Base())) {
2334 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMS, RC: &ARM::SPRRegClass, Op0);
2335 }
2336 return 0;
2337}
2338
2339unsigned fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, unsigned Op0) {
2340 if (RetVT.SimpleTy != MVT::f64)
2341 return 0;
2342 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2343 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMD, RC: &ARM::DPRRegClass, Op0);
2344 }
2345 return 0;
2346}
2347
2348unsigned fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
2349 if (RetVT.SimpleTy != MVT::v8f16)
2350 return 0;
2351 if ((Subtarget->hasMVEFloatOps())) {
2352 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16M, RC: &ARM::MQPRRegClass, Op0);
2353 }
2354 return 0;
2355}
2356
2357unsigned fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
2358 if (RetVT.SimpleTy != MVT::v4f32)
2359 return 0;
2360 if ((Subtarget->hasMVEFloatOps())) {
2361 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32M, RC: &ARM::MQPRRegClass, Op0);
2362 }
2363 return 0;
2364}
2365
2366unsigned fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, unsigned Op0) {
2367 switch (VT.SimpleTy) {
2368 case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0);
2369 case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0);
2370 case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0);
2371 case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0);
2372 case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0);
2373 default: return 0;
2374 }
2375}
2376
2377// FastEmit functions for ISD::FNEARBYINT.
2378
2379unsigned fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, unsigned Op0) {
2380 if (RetVT.SimpleTy != MVT::f16)
2381 return 0;
2382 if ((Subtarget->hasFullFP16())) {
2383 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRH, RC: &ARM::HPRRegClass, Op0);
2384 }
2385 return 0;
2386}
2387
2388unsigned fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, unsigned Op0) {
2389 if (RetVT.SimpleTy != MVT::f32)
2390 return 0;
2391 if ((Subtarget->hasFPARMv8Base())) {
2392 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRS, RC: &ARM::SPRRegClass, Op0);
2393 }
2394 return 0;
2395}
2396
2397unsigned fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, unsigned Op0) {
2398 if (RetVT.SimpleTy != MVT::f64)
2399 return 0;
2400 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2401 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRD, RC: &ARM::DPRRegClass, Op0);
2402 }
2403 return 0;
2404}
2405
2406unsigned fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, unsigned Op0) {
2407 switch (VT.SimpleTy) {
2408 case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0);
2409 case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0);
2410 case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0);
2411 default: return 0;
2412 }
2413}
2414
2415// FastEmit functions for ISD::FNEG.
2416
2417unsigned fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, unsigned Op0) {
2418 if (RetVT.SimpleTy != MVT::f16)
2419 return 0;
2420 if ((Subtarget->hasFullFP16())) {
2421 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGH, RC: &ARM::HPRRegClass, Op0);
2422 }
2423 return 0;
2424}
2425
2426unsigned fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, unsigned Op0) {
2427 if (RetVT.SimpleTy != MVT::f32)
2428 return 0;
2429 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
2430 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGS, RC: &ARM::SPRRegClass, Op0);
2431 }
2432 return 0;
2433}
2434
2435unsigned fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, unsigned Op0) {
2436 if (RetVT.SimpleTy != MVT::f64)
2437 return 0;
2438 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2439 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGD, RC: &ARM::DPRRegClass, Op0);
2440 }
2441 return 0;
2442}
2443
2444unsigned fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
2445 if (RetVT.SimpleTy != MVT::v4f16)
2446 return 0;
2447 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2448 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGhd, RC: &ARM::DPRRegClass, Op0);
2449 }
2450 return 0;
2451}
2452
2453unsigned fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
2454 if (RetVT.SimpleTy != MVT::v8f16)
2455 return 0;
2456 if ((Subtarget->hasMVEIntegerOps())) {
2457 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VNEGf16, RC: &ARM::MQPRRegClass, Op0);
2458 }
2459 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2460 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGhq, RC: &ARM::QPRRegClass, Op0);
2461 }
2462 return 0;
2463}
2464
2465unsigned fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
2466 if (RetVT.SimpleTy != MVT::v2f32)
2467 return 0;
2468 if ((Subtarget->hasNEON())) {
2469 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGfd, RC: &ARM::DPRRegClass, Op0);
2470 }
2471 return 0;
2472}
2473
2474unsigned fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
2475 if (RetVT.SimpleTy != MVT::v4f32)
2476 return 0;
2477 if ((Subtarget->hasMVEIntegerOps())) {
2478 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VNEGf32, RC: &ARM::MQPRRegClass, Op0);
2479 }
2480 if ((Subtarget->hasNEON())) {
2481 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGf32q, RC: &ARM::QPRRegClass, Op0);
2482 }
2483 return 0;
2484}
2485
2486unsigned fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, unsigned Op0) {
2487 switch (VT.SimpleTy) {
2488 case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0);
2489 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
2490 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
2491 case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0);
2492 case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0);
2493 case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0);
2494 case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0);
2495 default: return 0;
2496 }
2497}
2498
2499// FastEmit functions for ISD::FP_EXTEND.
2500
2501unsigned fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0) {
2502 if (RetVT.SimpleTy != MVT::f64)
2503 return 0;
2504 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2505 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTDS, RC: &ARM::DPRRegClass, Op0);
2506 }
2507 return 0;
2508}
2509
2510unsigned fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
2511 if (RetVT.SimpleTy != MVT::v4f32)
2512 return 0;
2513 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2f, RC: &ARM::QPRRegClass, Op0);
2514}
2515
2516unsigned fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) {
2517 switch (VT.SimpleTy) {
2518 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
2519 case MVT::v4f16: return fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
2520 default: return 0;
2521 }
2522}
2523
2524// FastEmit functions for ISD::FP_ROUND.
2525
2526unsigned fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0) {
2527 if (RetVT.SimpleTy != MVT::f32)
2528 return 0;
2529 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2530 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTSD, RC: &ARM::SPRRegClass, Op0);
2531 }
2532 return 0;
2533}
2534
2535unsigned fastEmit_ISD_FP_ROUND_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
2536 if (RetVT.SimpleTy != MVT::v4f16)
2537 return 0;
2538 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2h, RC: &ARM::DPRRegClass, Op0);
2539}
2540
2541unsigned fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0) {
2542 switch (VT.SimpleTy) {
2543 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
2544 case MVT::v4f32: return fastEmit_ISD_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
2545 default: return 0;
2546 }
2547}
2548
2549// FastEmit functions for ISD::FP_TO_SINT.
2550
2551unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
2552 if (RetVT.SimpleTy != MVT::v4i16)
2553 return 0;
2554 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2555 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2sd, RC: &ARM::DPRRegClass, Op0);
2556 }
2557 return 0;
2558}
2559
2560unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
2561 if (RetVT.SimpleTy != MVT::v8i16)
2562 return 0;
2563 if ((Subtarget->hasMVEFloatOps())) {
2564 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTs16f16z, RC: &ARM::MQPRRegClass, Op0);
2565 }
2566 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2567 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2sq, RC: &ARM::QPRRegClass, Op0);
2568 }
2569 return 0;
2570}
2571
2572unsigned fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
2573 if (RetVT.SimpleTy != MVT::v2i32)
2574 return 0;
2575 if ((Subtarget->hasNEON())) {
2576 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2sd, RC: &ARM::DPRRegClass, Op0);
2577 }
2578 return 0;
2579}
2580
2581unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
2582 if (RetVT.SimpleTy != MVT::v4i32)
2583 return 0;
2584 if ((Subtarget->hasMVEFloatOps())) {
2585 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTs32f32z, RC: &ARM::MQPRRegClass, Op0);
2586 }
2587 if ((Subtarget->hasNEON())) {
2588 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2sq, RC: &ARM::QPRRegClass, Op0);
2589 }
2590 return 0;
2591}
2592
2593unsigned fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0) {
2594 switch (VT.SimpleTy) {
2595 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
2596 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
2597 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
2598 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
2599 default: return 0;
2600 }
2601}
2602
2603// FastEmit functions for ISD::FP_TO_UINT.
2604
2605unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
2606 if (RetVT.SimpleTy != MVT::v4i16)
2607 return 0;
2608 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2609 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2ud, RC: &ARM::DPRRegClass, Op0);
2610 }
2611 return 0;
2612}
2613
2614unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
2615 if (RetVT.SimpleTy != MVT::v8i16)
2616 return 0;
2617 if ((Subtarget->hasMVEFloatOps())) {
2618 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTu16f16z, RC: &ARM::MQPRRegClass, Op0);
2619 }
2620 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2621 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2uq, RC: &ARM::QPRRegClass, Op0);
2622 }
2623 return 0;
2624}
2625
2626unsigned fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
2627 if (RetVT.SimpleTy != MVT::v2i32)
2628 return 0;
2629 if ((Subtarget->hasNEON())) {
2630 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2ud, RC: &ARM::DPRRegClass, Op0);
2631 }
2632 return 0;
2633}
2634
2635unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
2636 if (RetVT.SimpleTy != MVT::v4i32)
2637 return 0;
2638 if ((Subtarget->hasMVEFloatOps())) {
2639 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTu32f32z, RC: &ARM::MQPRRegClass, Op0);
2640 }
2641 if ((Subtarget->hasNEON())) {
2642 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2uq, RC: &ARM::QPRRegClass, Op0);
2643 }
2644 return 0;
2645}
2646
2647unsigned fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0) {
2648 switch (VT.SimpleTy) {
2649 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
2650 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
2651 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
2652 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
2653 default: return 0;
2654 }
2655}
2656
2657// FastEmit functions for ISD::FRINT.
2658
2659unsigned fastEmit_ISD_FRINT_MVT_f16_r(MVT RetVT, unsigned Op0) {
2660 if (RetVT.SimpleTy != MVT::f16)
2661 return 0;
2662 if ((Subtarget->hasFullFP16())) {
2663 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXH, RC: &ARM::HPRRegClass, Op0);
2664 }
2665 return 0;
2666}
2667
2668unsigned fastEmit_ISD_FRINT_MVT_f32_r(MVT RetVT, unsigned Op0) {
2669 if (RetVT.SimpleTy != MVT::f32)
2670 return 0;
2671 if ((Subtarget->hasFPARMv8Base())) {
2672 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXS, RC: &ARM::SPRRegClass, Op0);
2673 }
2674 return 0;
2675}
2676
2677unsigned fastEmit_ISD_FRINT_MVT_f64_r(MVT RetVT, unsigned Op0) {
2678 if (RetVT.SimpleTy != MVT::f64)
2679 return 0;
2680 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2681 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXD, RC: &ARM::DPRRegClass, Op0);
2682 }
2683 return 0;
2684}
2685
2686unsigned fastEmit_ISD_FRINT_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
2687 if (RetVT.SimpleTy != MVT::v8f16)
2688 return 0;
2689 if ((Subtarget->hasMVEFloatOps())) {
2690 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16X, RC: &ARM::MQPRRegClass, Op0);
2691 }
2692 return 0;
2693}
2694
2695unsigned fastEmit_ISD_FRINT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
2696 if (RetVT.SimpleTy != MVT::v4f32)
2697 return 0;
2698 if ((Subtarget->hasMVEFloatOps())) {
2699 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32X, RC: &ARM::MQPRRegClass, Op0);
2700 }
2701 return 0;
2702}
2703
2704unsigned fastEmit_ISD_FRINT_r(MVT VT, MVT RetVT, unsigned Op0) {
2705 switch (VT.SimpleTy) {
2706 case MVT::f16: return fastEmit_ISD_FRINT_MVT_f16_r(RetVT, Op0);
2707 case MVT::f32: return fastEmit_ISD_FRINT_MVT_f32_r(RetVT, Op0);
2708 case MVT::f64: return fastEmit_ISD_FRINT_MVT_f64_r(RetVT, Op0);
2709 case MVT::v8f16: return fastEmit_ISD_FRINT_MVT_v8f16_r(RetVT, Op0);
2710 case MVT::v4f32: return fastEmit_ISD_FRINT_MVT_v4f32_r(RetVT, Op0);
2711 default: return 0;
2712 }
2713}
2714
2715// FastEmit functions for ISD::FROUND.
2716
2717unsigned fastEmit_ISD_FROUND_MVT_f16_r(MVT RetVT, unsigned Op0) {
2718 if (RetVT.SimpleTy != MVT::f16)
2719 return 0;
2720 if ((Subtarget->hasFullFP16())) {
2721 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAH, RC: &ARM::HPRRegClass, Op0);
2722 }
2723 return 0;
2724}
2725
2726unsigned fastEmit_ISD_FROUND_MVT_f32_r(MVT RetVT, unsigned Op0) {
2727 if (RetVT.SimpleTy != MVT::f32)
2728 return 0;
2729 if ((Subtarget->hasFPARMv8Base())) {
2730 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAS, RC: &ARM::SPRRegClass, Op0);
2731 }
2732 return 0;
2733}
2734
2735unsigned fastEmit_ISD_FROUND_MVT_f64_r(MVT RetVT, unsigned Op0) {
2736 if (RetVT.SimpleTy != MVT::f64)
2737 return 0;
2738 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2739 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAD, RC: &ARM::DPRRegClass, Op0);
2740 }
2741 return 0;
2742}
2743
2744unsigned fastEmit_ISD_FROUND_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
2745 if (RetVT.SimpleTy != MVT::v8f16)
2746 return 0;
2747 if ((Subtarget->hasMVEFloatOps())) {
2748 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16A, RC: &ARM::MQPRRegClass, Op0);
2749 }
2750 return 0;
2751}
2752
2753unsigned fastEmit_ISD_FROUND_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
2754 if (RetVT.SimpleTy != MVT::v4f32)
2755 return 0;
2756 if ((Subtarget->hasMVEFloatOps())) {
2757 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32A, RC: &ARM::MQPRRegClass, Op0);
2758 }
2759 return 0;
2760}
2761
2762unsigned fastEmit_ISD_FROUND_r(MVT VT, MVT RetVT, unsigned Op0) {
2763 switch (VT.SimpleTy) {
2764 case MVT::f16: return fastEmit_ISD_FROUND_MVT_f16_r(RetVT, Op0);
2765 case MVT::f32: return fastEmit_ISD_FROUND_MVT_f32_r(RetVT, Op0);
2766 case MVT::f64: return fastEmit_ISD_FROUND_MVT_f64_r(RetVT, Op0);
2767 case MVT::v8f16: return fastEmit_ISD_FROUND_MVT_v8f16_r(RetVT, Op0);
2768 case MVT::v4f32: return fastEmit_ISD_FROUND_MVT_v4f32_r(RetVT, Op0);
2769 default: return 0;
2770 }
2771}
2772
2773// FastEmit functions for ISD::FSQRT.
2774
2775unsigned fastEmit_ISD_FSQRT_MVT_f16_r(MVT RetVT, unsigned Op0) {
2776 if (RetVT.SimpleTy != MVT::f16)
2777 return 0;
2778 if ((Subtarget->hasFullFP16())) {
2779 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTH, RC: &ARM::HPRRegClass, Op0);
2780 }
2781 return 0;
2782}
2783
2784unsigned fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0) {
2785 if (RetVT.SimpleTy != MVT::f32)
2786 return 0;
2787 if ((Subtarget->hasVFP2Base())) {
2788 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTS, RC: &ARM::SPRRegClass, Op0);
2789 }
2790 return 0;
2791}
2792
2793unsigned fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0) {
2794 if (RetVT.SimpleTy != MVT::f64)
2795 return 0;
2796 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2797 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTD, RC: &ARM::DPRRegClass, Op0);
2798 }
2799 return 0;
2800}
2801
2802unsigned fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0) {
2803 switch (VT.SimpleTy) {
2804 case MVT::f16: return fastEmit_ISD_FSQRT_MVT_f16_r(RetVT, Op0);
2805 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
2806 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
2807 default: return 0;
2808 }
2809}
2810
2811// FastEmit functions for ISD::FTRUNC.
2812
2813unsigned fastEmit_ISD_FTRUNC_MVT_f16_r(MVT RetVT, unsigned Op0) {
2814 if (RetVT.SimpleTy != MVT::f16)
2815 return 0;
2816 if ((Subtarget->hasFullFP16())) {
2817 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZH, RC: &ARM::HPRRegClass, Op0);
2818 }
2819 return 0;
2820}
2821
2822unsigned fastEmit_ISD_FTRUNC_MVT_f32_r(MVT RetVT, unsigned Op0) {
2823 if (RetVT.SimpleTy != MVT::f32)
2824 return 0;
2825 if ((Subtarget->hasFPARMv8Base())) {
2826 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZS, RC: &ARM::SPRRegClass, Op0);
2827 }
2828 return 0;
2829}
2830
2831unsigned fastEmit_ISD_FTRUNC_MVT_f64_r(MVT RetVT, unsigned Op0) {
2832 if (RetVT.SimpleTy != MVT::f64)
2833 return 0;
2834 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2835 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZD, RC: &ARM::DPRRegClass, Op0);
2836 }
2837 return 0;
2838}
2839
2840unsigned fastEmit_ISD_FTRUNC_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
2841 if (RetVT.SimpleTy != MVT::v8f16)
2842 return 0;
2843 if ((Subtarget->hasMVEFloatOps())) {
2844 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16Z, RC: &ARM::MQPRRegClass, Op0);
2845 }
2846 return 0;
2847}
2848
2849unsigned fastEmit_ISD_FTRUNC_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
2850 if (RetVT.SimpleTy != MVT::v4f32)
2851 return 0;
2852 if ((Subtarget->hasMVEFloatOps())) {
2853 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32Z, RC: &ARM::MQPRRegClass, Op0);
2854 }
2855 return 0;
2856}
2857
2858unsigned fastEmit_ISD_FTRUNC_r(MVT VT, MVT RetVT, unsigned Op0) {
2859 switch (VT.SimpleTy) {
2860 case MVT::f16: return fastEmit_ISD_FTRUNC_MVT_f16_r(RetVT, Op0);
2861 case MVT::f32: return fastEmit_ISD_FTRUNC_MVT_f32_r(RetVT, Op0);
2862 case MVT::f64: return fastEmit_ISD_FTRUNC_MVT_f64_r(RetVT, Op0);
2863 case MVT::v8f16: return fastEmit_ISD_FTRUNC_MVT_v8f16_r(RetVT, Op0);
2864 case MVT::v4f32: return fastEmit_ISD_FTRUNC_MVT_v4f32_r(RetVT, Op0);
2865 default: return 0;
2866 }
2867}
2868
2869// FastEmit functions for ISD::SET_FPENV.
2870
2871unsigned fastEmit_ISD_SET_FPENV_MVT_i32_r(MVT RetVT, unsigned Op0) {
2872 if (RetVT.SimpleTy != MVT::isVoid)
2873 return 0;
2874 return fastEmitInst_r(MachineInstOpcode: ARM::VMSR, RC: &ARM::GPRnopcRegClass, Op0);
2875}
2876
2877unsigned fastEmit_ISD_SET_FPENV_r(MVT VT, MVT RetVT, unsigned Op0) {
2878 switch (VT.SimpleTy) {
2879 case MVT::i32: return fastEmit_ISD_SET_FPENV_MVT_i32_r(RetVT, Op0);
2880 default: return 0;
2881 }
2882}
2883
2884// FastEmit functions for ISD::SIGN_EXTEND.
2885
2886unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
2887 if (RetVT.SimpleTy != MVT::v8i16)
2888 return 0;
2889 if ((Subtarget->hasNEON())) {
2890 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLsv8i16, RC: &ARM::QPRRegClass, Op0);
2891 }
2892 return 0;
2893}
2894
2895unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
2896 if (RetVT.SimpleTy != MVT::v4i32)
2897 return 0;
2898 if ((Subtarget->hasNEON())) {
2899 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLsv4i32, RC: &ARM::QPRRegClass, Op0);
2900 }
2901 return 0;
2902}
2903
2904unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
2905 if (RetVT.SimpleTy != MVT::v2i64)
2906 return 0;
2907 if ((Subtarget->hasNEON())) {
2908 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLsv2i64, RC: &ARM::QPRRegClass, Op0);
2909 }
2910 return 0;
2911}
2912
2913unsigned fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) {
2914 switch (VT.SimpleTy) {
2915 case MVT::v8i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i8_r(RetVT, Op0);
2916 case MVT::v4i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i16_r(RetVT, Op0);
2917 case MVT::v2i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i32_r(RetVT, Op0);
2918 default: return 0;
2919 }
2920}
2921
2922// FastEmit functions for ISD::SINT_TO_FP.
2923
2924unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
2925 if (RetVT.SimpleTy != MVT::v4f16)
2926 return 0;
2927 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2928 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2hd, RC: &ARM::DPRRegClass, Op0);
2929 }
2930 return 0;
2931}
2932
2933unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
2934 if (RetVT.SimpleTy != MVT::v8f16)
2935 return 0;
2936 if ((Subtarget->hasMVEFloatOps())) {
2937 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf16s16n, RC: &ARM::MQPRRegClass, Op0);
2938 }
2939 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2940 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2hq, RC: &ARM::QPRRegClass, Op0);
2941 }
2942 return 0;
2943}
2944
2945unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
2946 if (RetVT.SimpleTy != MVT::v2f32)
2947 return 0;
2948 if ((Subtarget->hasNEON())) {
2949 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2fd, RC: &ARM::DPRRegClass, Op0);
2950 }
2951 return 0;
2952}
2953
2954unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
2955 if (RetVT.SimpleTy != MVT::v4f32)
2956 return 0;
2957 if ((Subtarget->hasMVEFloatOps())) {
2958 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf32s32n, RC: &ARM::MQPRRegClass, Op0);
2959 }
2960 if ((Subtarget->hasNEON())) {
2961 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2fq, RC: &ARM::QPRRegClass, Op0);
2962 }
2963 return 0;
2964}
2965
2966unsigned fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0) {
2967 switch (VT.SimpleTy) {
2968 case MVT::v4i16: return fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
2969 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
2970 case MVT::v2i32: return fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
2971 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
2972 default: return 0;
2973 }
2974}
2975
2976// FastEmit functions for ISD::TRUNCATE.
2977
2978unsigned fastEmit_ISD_TRUNCATE_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
2979 if (RetVT.SimpleTy != MVT::v8i8)
2980 return 0;
2981 if ((Subtarget->hasNEON())) {
2982 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVNv8i8, RC: &ARM::DPRRegClass, Op0);
2983 }
2984 return 0;
2985}
2986
2987unsigned fastEmit_ISD_TRUNCATE_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
2988 if (RetVT.SimpleTy != MVT::v4i16)
2989 return 0;
2990 if ((Subtarget->hasNEON())) {
2991 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVNv4i16, RC: &ARM::DPRRegClass, Op0);
2992 }
2993 return 0;
2994}
2995
2996unsigned fastEmit_ISD_TRUNCATE_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
2997 if (RetVT.SimpleTy != MVT::v2i32)
2998 return 0;
2999 if ((Subtarget->hasNEON())) {
3000 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVNv2i32, RC: &ARM::DPRRegClass, Op0);
3001 }
3002 return 0;
3003}
3004
3005unsigned fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, unsigned Op0) {
3006 switch (VT.SimpleTy) {
3007 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i16_r(RetVT, Op0);
3008 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_MVT_v4i32_r(RetVT, Op0);
3009 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_MVT_v2i64_r(RetVT, Op0);
3010 default: return 0;
3011 }
3012}
3013
3014// FastEmit functions for ISD::UINT_TO_FP.
3015
3016unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
3017 if (RetVT.SimpleTy != MVT::v4f16)
3018 return 0;
3019 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3020 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2hd, RC: &ARM::DPRRegClass, Op0);
3021 }
3022 return 0;
3023}
3024
3025unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
3026 if (RetVT.SimpleTy != MVT::v8f16)
3027 return 0;
3028 if ((Subtarget->hasMVEFloatOps())) {
3029 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf16u16n, RC: &ARM::MQPRRegClass, Op0);
3030 }
3031 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3032 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2hq, RC: &ARM::QPRRegClass, Op0);
3033 }
3034 return 0;
3035}
3036
3037unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
3038 if (RetVT.SimpleTy != MVT::v2f32)
3039 return 0;
3040 if ((Subtarget->hasNEON())) {
3041 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2fd, RC: &ARM::DPRRegClass, Op0);
3042 }
3043 return 0;
3044}
3045
3046unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
3047 if (RetVT.SimpleTy != MVT::v4f32)
3048 return 0;
3049 if ((Subtarget->hasMVEFloatOps())) {
3050 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf32u32n, RC: &ARM::MQPRRegClass, Op0);
3051 }
3052 if ((Subtarget->hasNEON())) {
3053 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2fq, RC: &ARM::QPRRegClass, Op0);
3054 }
3055 return 0;
3056}
3057
3058unsigned fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0) {
3059 switch (VT.SimpleTy) {
3060 case MVT::v4i16: return fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
3061 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3062 case MVT::v2i32: return fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
3063 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3064 default: return 0;
3065 }
3066}
3067
3068// FastEmit functions for ISD::VECREDUCE_ADD.
3069
3070unsigned fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
3071 if (RetVT.SimpleTy != MVT::i32)
3072 return 0;
3073 if ((Subtarget->hasMVEIntegerOps())) {
3074 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu8no_acc, RC: &ARM::tGPREvenRegClass, Op0);
3075 }
3076 return 0;
3077}
3078
3079unsigned fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
3080 if (RetVT.SimpleTy != MVT::i32)
3081 return 0;
3082 if ((Subtarget->hasMVEIntegerOps())) {
3083 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu16no_acc, RC: &ARM::tGPREvenRegClass, Op0);
3084 }
3085 return 0;
3086}
3087
3088unsigned fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
3089 if (RetVT.SimpleTy != MVT::i32)
3090 return 0;
3091 if ((Subtarget->hasMVEIntegerOps())) {
3092 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu32no_acc, RC: &ARM::tGPREvenRegClass, Op0);
3093 }
3094 return 0;
3095}
3096
3097unsigned fastEmit_ISD_VECREDUCE_ADD_r(MVT VT, MVT RetVT, unsigned Op0) {
3098 switch (VT.SimpleTy) {
3099 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(RetVT, Op0);
3100 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(RetVT, Op0);
3101 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(RetVT, Op0);
3102 default: return 0;
3103 }
3104}
3105
3106// FastEmit functions for ISD::ZERO_EXTEND.
3107
3108unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
3109 if (RetVT.SimpleTy != MVT::v8i16)
3110 return 0;
3111 if ((Subtarget->hasNEON())) {
3112 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv8i16, RC: &ARM::QPRRegClass, Op0);
3113 }
3114 return 0;
3115}
3116
3117unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
3118 if (RetVT.SimpleTy != MVT::v4i32)
3119 return 0;
3120 if ((Subtarget->hasNEON())) {
3121 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv4i32, RC: &ARM::QPRRegClass, Op0);
3122 }
3123 return 0;
3124}
3125
3126unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
3127 if (RetVT.SimpleTy != MVT::v2i64)
3128 return 0;
3129 if ((Subtarget->hasNEON())) {
3130 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv2i64, RC: &ARM::QPRRegClass, Op0);
3131 }
3132 return 0;
3133}
3134
3135unsigned fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) {
3136 switch (VT.SimpleTy) {
3137 case MVT::v8i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i8_r(RetVT, Op0);
3138 case MVT::v4i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v4i16_r(RetVT, Op0);
3139 case MVT::v2i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v2i32_r(RetVT, Op0);
3140 default: return 0;
3141 }
3142}
3143
3144// Top-level FastEmit function.
3145
3146unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0) override {
3147 switch (Opcode) {
3148 case ARMISD::CALL: return fastEmit_ARMISD_CALL_r(VT, RetVT, Op0);
3149 case ARMISD::CALL_NOLINK: return fastEmit_ARMISD_CALL_NOLINK_r(VT, RetVT, Op0);
3150 case ARMISD::CALL_PRED: return fastEmit_ARMISD_CALL_PRED_r(VT, RetVT, Op0);
3151 case ARMISD::CMPFPEw0: return fastEmit_ARMISD_CMPFPEw0_r(VT, RetVT, Op0);
3152 case ARMISD::CMPFPw0: return fastEmit_ARMISD_CMPFPw0_r(VT, RetVT, Op0);
3153 case ARMISD::RRX: return fastEmit_ARMISD_RRX_r(VT, RetVT, Op0);
3154 case ARMISD::SRA_GLUE: return fastEmit_ARMISD_SRA_GLUE_r(VT, RetVT, Op0);
3155 case ARMISD::SRL_GLUE: return fastEmit_ARMISD_SRL_GLUE_r(VT, RetVT, Op0);
3156 case ARMISD::VADDVs: return fastEmit_ARMISD_VADDVs_r(VT, RetVT, Op0);
3157 case ARMISD::VADDVu: return fastEmit_ARMISD_VADDVu_r(VT, RetVT, Op0);
3158 case ARMISD::VDUP: return fastEmit_ARMISD_VDUP_r(VT, RetVT, Op0);
3159 case ARMISD::VMOVSR: return fastEmit_ARMISD_VMOVSR_r(VT, RetVT, Op0);
3160 case ARMISD::VMOVhr: return fastEmit_ARMISD_VMOVhr_r(VT, RetVT, Op0);
3161 case ARMISD::VMOVrh: return fastEmit_ARMISD_VMOVrh_r(VT, RetVT, Op0);
3162 case ARMISD::VREV16: return fastEmit_ARMISD_VREV16_r(VT, RetVT, Op0);
3163 case ARMISD::VREV32: return fastEmit_ARMISD_VREV32_r(VT, RetVT, Op0);
3164 case ARMISD::VREV64: return fastEmit_ARMISD_VREV64_r(VT, RetVT, Op0);
3165 case ARMISD::WIN__DBZCHK: return fastEmit_ARMISD_WIN__DBZCHK_r(VT, RetVT, Op0);
3166 case ARMISD::tSECALL: return fastEmit_ARMISD_tSECALL_r(VT, RetVT, Op0);
3167 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
3168 case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0);
3169 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
3170 case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0);
3171 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0);
3172 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
3173 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
3174 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
3175 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
3176 case ISD::FCEIL: return fastEmit_ISD_FCEIL_r(VT, RetVT, Op0);
3177 case ISD::FFLOOR: return fastEmit_ISD_FFLOOR_r(VT, RetVT, Op0);
3178 case ISD::FNEARBYINT: return fastEmit_ISD_FNEARBYINT_r(VT, RetVT, Op0);
3179 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
3180 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
3181 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
3182 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
3183 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
3184 case ISD::FRINT: return fastEmit_ISD_FRINT_r(VT, RetVT, Op0);
3185 case ISD::FROUND: return fastEmit_ISD_FROUND_r(VT, RetVT, Op0);
3186 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
3187 case ISD::FTRUNC: return fastEmit_ISD_FTRUNC_r(VT, RetVT, Op0);
3188 case ISD::SET_FPENV: return fastEmit_ISD_SET_FPENV_r(VT, RetVT, Op0);
3189 case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0);
3190 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
3191 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
3192 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
3193 case ISD::VECREDUCE_ADD: return fastEmit_ISD_VECREDUCE_ADD_r(VT, RetVT, Op0);
3194 case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0);
3195 default: return 0;
3196 }
3197}
3198
3199// FastEmit functions for ARMISD::CMP.
3200
3201unsigned fastEmit_ARMISD_CMP_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3202 if (RetVT.SimpleTy != MVT::isVoid)
3203 return 0;
3204 if ((Subtarget->isThumb2())) {
3205 return fastEmitInst_rr(MachineInstOpcode: ARM::t2CMPrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3206 }
3207 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
3208 return fastEmitInst_rr(MachineInstOpcode: ARM::tCMPr, RC: &ARM::tGPRRegClass, Op0, Op1);
3209 }
3210 if ((!Subtarget->isThumb())) {
3211 return fastEmitInst_rr(MachineInstOpcode: ARM::CMPrr, RC: &ARM::GPRRegClass, Op0, Op1);
3212 }
3213 return 0;
3214}
3215
3216unsigned fastEmit_ARMISD_CMP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3217 switch (VT.SimpleTy) {
3218 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_rr(RetVT, Op0, Op1);
3219 default: return 0;
3220 }
3221}
3222
3223// FastEmit functions for ARMISD::CMPFP.
3224
3225unsigned fastEmit_ARMISD_CMPFP_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3226 if (RetVT.SimpleTy != MVT::isVoid)
3227 return 0;
3228 if ((Subtarget->hasFullFP16())) {
3229 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPH, RC: &ARM::HPRRegClass, Op0, Op1);
3230 }
3231 return 0;
3232}
3233
3234unsigned fastEmit_ARMISD_CMPFP_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3235 if (RetVT.SimpleTy != MVT::isVoid)
3236 return 0;
3237 if ((Subtarget->hasVFP2Base())) {
3238 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPS, RC: &ARM::SPRRegClass, Op0, Op1);
3239 }
3240 return 0;
3241}
3242
3243unsigned fastEmit_ARMISD_CMPFP_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3244 if (RetVT.SimpleTy != MVT::isVoid)
3245 return 0;
3246 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3247 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPD, RC: &ARM::DPRRegClass, Op0, Op1);
3248 }
3249 return 0;
3250}
3251
3252unsigned fastEmit_ARMISD_CMPFP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3253 switch (VT.SimpleTy) {
3254 case MVT::f16: return fastEmit_ARMISD_CMPFP_MVT_f16_rr(RetVT, Op0, Op1);
3255 case MVT::f32: return fastEmit_ARMISD_CMPFP_MVT_f32_rr(RetVT, Op0, Op1);
3256 case MVT::f64: return fastEmit_ARMISD_CMPFP_MVT_f64_rr(RetVT, Op0, Op1);
3257 default: return 0;
3258 }
3259}
3260
3261// FastEmit functions for ARMISD::CMPFPE.
3262
3263unsigned fastEmit_ARMISD_CMPFPE_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3264 if (RetVT.SimpleTy != MVT::isVoid)
3265 return 0;
3266 if ((Subtarget->hasFullFP16())) {
3267 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPEH, RC: &ARM::HPRRegClass, Op0, Op1);
3268 }
3269 return 0;
3270}
3271
3272unsigned fastEmit_ARMISD_CMPFPE_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3273 if (RetVT.SimpleTy != MVT::isVoid)
3274 return 0;
3275 if ((Subtarget->hasVFP2Base())) {
3276 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPES, RC: &ARM::SPRRegClass, Op0, Op1);
3277 }
3278 return 0;
3279}
3280
3281unsigned fastEmit_ARMISD_CMPFPE_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3282 if (RetVT.SimpleTy != MVT::isVoid)
3283 return 0;
3284 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3285 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPED, RC: &ARM::DPRRegClass, Op0, Op1);
3286 }
3287 return 0;
3288}
3289
3290unsigned fastEmit_ARMISD_CMPFPE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3291 switch (VT.SimpleTy) {
3292 case MVT::f16: return fastEmit_ARMISD_CMPFPE_MVT_f16_rr(RetVT, Op0, Op1);
3293 case MVT::f32: return fastEmit_ARMISD_CMPFPE_MVT_f32_rr(RetVT, Op0, Op1);
3294 case MVT::f64: return fastEmit_ARMISD_CMPFPE_MVT_f64_rr(RetVT, Op0, Op1);
3295 default: return 0;
3296 }
3297}
3298
3299// FastEmit functions for ARMISD::CMPZ.
3300
3301unsigned fastEmit_ARMISD_CMPZ_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3302 if (RetVT.SimpleTy != MVT::isVoid)
3303 return 0;
3304 if ((Subtarget->isThumb2())) {
3305 return fastEmitInst_rr(MachineInstOpcode: ARM::t2CMPrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3306 }
3307 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
3308 return fastEmitInst_rr(MachineInstOpcode: ARM::tCMPr, RC: &ARM::tGPRRegClass, Op0, Op1);
3309 }
3310 if ((!Subtarget->isThumb())) {
3311 return fastEmitInst_rr(MachineInstOpcode: ARM::CMPrr, RC: &ARM::GPRRegClass, Op0, Op1);
3312 }
3313 return 0;
3314}
3315
3316unsigned fastEmit_ARMISD_CMPZ_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3317 switch (VT.SimpleTy) {
3318 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_rr(RetVT, Op0, Op1);
3319 default: return 0;
3320 }
3321}
3322
3323// FastEmit functions for ARMISD::EH_SJLJ_LONGJMP.
3324
3325unsigned fastEmit_ARMISD_EH_SJLJ_LONGJMP_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3326 if (RetVT.SimpleTy != MVT::isVoid)
3327 return 0;
3328 if ((Subtarget->isThumb()) && (Subtarget->isTargetWindows())) {
3329 return fastEmitInst_rr(MachineInstOpcode: ARM::tInt_WIN_eh_sjlj_longjmp, RC: &ARM::GPRRegClass, Op0, Op1);
3330 }
3331 if ((!Subtarget->isTargetWindows()) && (Subtarget->isThumb())) {
3332 return fastEmitInst_rr(MachineInstOpcode: ARM::tInt_eh_sjlj_longjmp, RC: &ARM::tGPRRegClass, Op0, Op1);
3333 }
3334 if ((!Subtarget->isThumb())) {
3335 return fastEmitInst_rr(MachineInstOpcode: ARM::Int_eh_sjlj_longjmp, RC: &ARM::GPRRegClass, Op0, Op1);
3336 }
3337 return 0;
3338}
3339
3340unsigned fastEmit_ARMISD_EH_SJLJ_LONGJMP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3341 switch (VT.SimpleTy) {
3342 case MVT::i32: return fastEmit_ARMISD_EH_SJLJ_LONGJMP_MVT_i32_rr(RetVT, Op0, Op1);
3343 default: return 0;
3344 }
3345}
3346
3347// FastEmit functions for ARMISD::EH_SJLJ_SETJMP.
3348
3349unsigned fastEmit_ARMISD_EH_SJLJ_SETJMP_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3350 if (RetVT.SimpleTy != MVT::i32)
3351 return 0;
3352 if ((Subtarget->isThumb2()) && (!Subtarget->hasVFP2Base())) {
3353 return fastEmitInst_rr(MachineInstOpcode: ARM::t2Int_eh_sjlj_setjmp_nofp, RC: &ARM::tGPRRegClass, Op0, Op1);
3354 }
3355 if ((Subtarget->hasVFP2Base()) && (Subtarget->isThumb2())) {
3356 return fastEmitInst_rr(MachineInstOpcode: ARM::t2Int_eh_sjlj_setjmp, RC: &ARM::tGPRRegClass, Op0, Op1);
3357 }
3358 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
3359 return fastEmitInst_rr(MachineInstOpcode: ARM::tInt_eh_sjlj_setjmp, RC: &ARM::tGPRRegClass, Op0, Op1);
3360 }
3361 if ((!Subtarget->isThumb()) && (!Subtarget->hasVFP2Base())) {
3362 return fastEmitInst_rr(MachineInstOpcode: ARM::Int_eh_sjlj_setjmp_nofp, RC: &ARM::GPRRegClass, Op0, Op1);
3363 }
3364 if ((Subtarget->hasVFP2Base()) && (!Subtarget->isThumb())) {
3365 return fastEmitInst_rr(MachineInstOpcode: ARM::Int_eh_sjlj_setjmp, RC: &ARM::GPRRegClass, Op0, Op1);
3366 }
3367 return 0;
3368}
3369
3370unsigned fastEmit_ARMISD_EH_SJLJ_SETJMP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3371 switch (VT.SimpleTy) {
3372 case MVT::i32: return fastEmit_ARMISD_EH_SJLJ_SETJMP_MVT_i32_rr(RetVT, Op0, Op1);
3373 default: return 0;
3374 }
3375}
3376
3377// FastEmit functions for ARMISD::QADD16b.
3378
3379unsigned fastEmit_ARMISD_QADD16b_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3380 if (RetVT.SimpleTy != MVT::i32)
3381 return 0;
3382 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3383 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QADD16, RC: &ARM::rGPRRegClass, Op0, Op1);
3384 }
3385 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3386 return fastEmitInst_rr(MachineInstOpcode: ARM::QADD16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3387 }
3388 return 0;
3389}
3390
3391unsigned fastEmit_ARMISD_QADD16b_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3392 switch (VT.SimpleTy) {
3393 case MVT::i32: return fastEmit_ARMISD_QADD16b_MVT_i32_rr(RetVT, Op0, Op1);
3394 default: return 0;
3395 }
3396}
3397
3398// FastEmit functions for ARMISD::QADD8b.
3399
3400unsigned fastEmit_ARMISD_QADD8b_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3401 if (RetVT.SimpleTy != MVT::i32)
3402 return 0;
3403 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3404 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QADD8, RC: &ARM::rGPRRegClass, Op0, Op1);
3405 }
3406 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3407 return fastEmitInst_rr(MachineInstOpcode: ARM::QADD8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3408 }
3409 return 0;
3410}
3411
3412unsigned fastEmit_ARMISD_QADD8b_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3413 switch (VT.SimpleTy) {
3414 case MVT::i32: return fastEmit_ARMISD_QADD8b_MVT_i32_rr(RetVT, Op0, Op1);
3415 default: return 0;
3416 }
3417}
3418
3419// FastEmit functions for ARMISD::QSUB16b.
3420
3421unsigned fastEmit_ARMISD_QSUB16b_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3422 if (RetVT.SimpleTy != MVT::i32)
3423 return 0;
3424 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3425 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QSUB16, RC: &ARM::rGPRRegClass, Op0, Op1);
3426 }
3427 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3428 return fastEmitInst_rr(MachineInstOpcode: ARM::QSUB16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3429 }
3430 return 0;
3431}
3432
3433unsigned fastEmit_ARMISD_QSUB16b_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3434 switch (VT.SimpleTy) {
3435 case MVT::i32: return fastEmit_ARMISD_QSUB16b_MVT_i32_rr(RetVT, Op0, Op1);
3436 default: return 0;
3437 }
3438}
3439
3440// FastEmit functions for ARMISD::QSUB8b.
3441
3442unsigned fastEmit_ARMISD_QSUB8b_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3443 if (RetVT.SimpleTy != MVT::i32)
3444 return 0;
3445 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3446 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QSUB8, RC: &ARM::rGPRRegClass, Op0, Op1);
3447 }
3448 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3449 return fastEmitInst_rr(MachineInstOpcode: ARM::QSUB8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3450 }
3451 return 0;
3452}
3453
3454unsigned fastEmit_ARMISD_QSUB8b_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3455 switch (VT.SimpleTy) {
3456 case MVT::i32: return fastEmit_ARMISD_QSUB8b_MVT_i32_rr(RetVT, Op0, Op1);
3457 default: return 0;
3458 }
3459}
3460
3461// FastEmit functions for ARMISD::SMULWB.
3462
3463unsigned fastEmit_ARMISD_SMULWB_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3464 if (RetVT.SimpleTy != MVT::i32)
3465 return 0;
3466 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3467 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SMULWB, RC: &ARM::rGPRRegClass, Op0, Op1);
3468 }
3469 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
3470 return fastEmitInst_rr(MachineInstOpcode: ARM::SMULWB, RC: &ARM::GPRRegClass, Op0, Op1);
3471 }
3472 return 0;
3473}
3474
3475unsigned fastEmit_ARMISD_SMULWB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3476 switch (VT.SimpleTy) {
3477 case MVT::i32: return fastEmit_ARMISD_SMULWB_MVT_i32_rr(RetVT, Op0, Op1);
3478 default: return 0;
3479 }
3480}
3481
3482// FastEmit functions for ARMISD::SMULWT.
3483
3484unsigned fastEmit_ARMISD_SMULWT_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3485 if (RetVT.SimpleTy != MVT::i32)
3486 return 0;
3487 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3488 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SMULWT, RC: &ARM::rGPRRegClass, Op0, Op1);
3489 }
3490 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
3491 return fastEmitInst_rr(MachineInstOpcode: ARM::SMULWT, RC: &ARM::GPRRegClass, Op0, Op1);
3492 }
3493 return 0;
3494}
3495
3496unsigned fastEmit_ARMISD_SMULWT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3497 switch (VT.SimpleTy) {
3498 case MVT::i32: return fastEmit_ARMISD_SMULWT_MVT_i32_rr(RetVT, Op0, Op1);
3499 default: return 0;
3500 }
3501}
3502
3503// FastEmit functions for ARMISD::UQADD16b.
3504
3505unsigned fastEmit_ARMISD_UQADD16b_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3506 if (RetVT.SimpleTy != MVT::i32)
3507 return 0;
3508 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3509 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQADD16, RC: &ARM::rGPRRegClass, Op0, Op1);
3510 }
3511 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3512 return fastEmitInst_rr(MachineInstOpcode: ARM::UQADD16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3513 }
3514 return 0;
3515}
3516
3517unsigned fastEmit_ARMISD_UQADD16b_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3518 switch (VT.SimpleTy) {
3519 case MVT::i32: return fastEmit_ARMISD_UQADD16b_MVT_i32_rr(RetVT, Op0, Op1);
3520 default: return 0;
3521 }
3522}
3523
3524// FastEmit functions for ARMISD::UQADD8b.
3525
3526unsigned fastEmit_ARMISD_UQADD8b_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3527 if (RetVT.SimpleTy != MVT::i32)
3528 return 0;
3529 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3530 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQADD8, RC: &ARM::rGPRRegClass, Op0, Op1);
3531 }
3532 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3533 return fastEmitInst_rr(MachineInstOpcode: ARM::UQADD8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3534 }
3535 return 0;
3536}
3537
3538unsigned fastEmit_ARMISD_UQADD8b_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3539 switch (VT.SimpleTy) {
3540 case MVT::i32: return fastEmit_ARMISD_UQADD8b_MVT_i32_rr(RetVT, Op0, Op1);
3541 default: return 0;
3542 }
3543}
3544
3545// FastEmit functions for ARMISD::UQSUB16b.
3546
3547unsigned fastEmit_ARMISD_UQSUB16b_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3548 if (RetVT.SimpleTy != MVT::i32)
3549 return 0;
3550 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3551 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQSUB16, RC: &ARM::rGPRRegClass, Op0, Op1);
3552 }
3553 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3554 return fastEmitInst_rr(MachineInstOpcode: ARM::UQSUB16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3555 }
3556 return 0;
3557}
3558
3559unsigned fastEmit_ARMISD_UQSUB16b_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3560 switch (VT.SimpleTy) {
3561 case MVT::i32: return fastEmit_ARMISD_UQSUB16b_MVT_i32_rr(RetVT, Op0, Op1);
3562 default: return 0;
3563 }
3564}
3565
3566// FastEmit functions for ARMISD::UQSUB8b.
3567
3568unsigned fastEmit_ARMISD_UQSUB8b_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3569 if (RetVT.SimpleTy != MVT::i32)
3570 return 0;
3571 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3572 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQSUB8, RC: &ARM::rGPRRegClass, Op0, Op1);
3573 }
3574 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3575 return fastEmitInst_rr(MachineInstOpcode: ARM::UQSUB8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
3576 }
3577 return 0;
3578}
3579
3580unsigned fastEmit_ARMISD_UQSUB8b_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3581 switch (VT.SimpleTy) {
3582 case MVT::i32: return fastEmit_ARMISD_UQSUB8b_MVT_i32_rr(RetVT, Op0, Op1);
3583 default: return 0;
3584 }
3585}
3586
3587// FastEmit functions for ARMISD::VMLAVs.
3588
3589unsigned fastEmit_ARMISD_VMLAVs_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3590 if (RetVT.SimpleTy != MVT::i32)
3591 return 0;
3592 if ((Subtarget->hasMVEIntegerOps())) {
3593 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVs8, RC: &ARM::tGPREvenRegClass, Op0, Op1);
3594 }
3595 return 0;
3596}
3597
3598unsigned fastEmit_ARMISD_VMLAVs_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3599 if (RetVT.SimpleTy != MVT::i32)
3600 return 0;
3601 if ((Subtarget->hasMVEIntegerOps())) {
3602 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVs16, RC: &ARM::tGPREvenRegClass, Op0, Op1);
3603 }
3604 return 0;
3605}
3606
3607unsigned fastEmit_ARMISD_VMLAVs_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3608 switch (VT.SimpleTy) {
3609 case MVT::v16i8: return fastEmit_ARMISD_VMLAVs_MVT_v16i8_rr(RetVT, Op0, Op1);
3610 case MVT::v8i16: return fastEmit_ARMISD_VMLAVs_MVT_v8i16_rr(RetVT, Op0, Op1);
3611 default: return 0;
3612 }
3613}
3614
3615// FastEmit functions for ARMISD::VMLAVu.
3616
3617unsigned fastEmit_ARMISD_VMLAVu_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3618 if (RetVT.SimpleTy != MVT::i32)
3619 return 0;
3620 if ((Subtarget->hasMVEIntegerOps())) {
3621 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVu8, RC: &ARM::tGPREvenRegClass, Op0, Op1);
3622 }
3623 return 0;
3624}
3625
3626unsigned fastEmit_ARMISD_VMLAVu_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3627 if (RetVT.SimpleTy != MVT::i32)
3628 return 0;
3629 if ((Subtarget->hasMVEIntegerOps())) {
3630 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVu16, RC: &ARM::tGPREvenRegClass, Op0, Op1);
3631 }
3632 return 0;
3633}
3634
3635unsigned fastEmit_ARMISD_VMLAVu_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3636 switch (VT.SimpleTy) {
3637 case MVT::v16i8: return fastEmit_ARMISD_VMLAVu_MVT_v16i8_rr(RetVT, Op0, Op1);
3638 case MVT::v8i16: return fastEmit_ARMISD_VMLAVu_MVT_v8i16_rr(RetVT, Op0, Op1);
3639 default: return 0;
3640 }
3641}
3642
3643// FastEmit functions for ARMISD::VMOVDRR.
3644
3645unsigned fastEmit_ARMISD_VMOVDRR_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3646 if (RetVT.SimpleTy != MVT::f64)
3647 return 0;
3648 if ((Subtarget->hasFPRegs())) {
3649 return fastEmitInst_rr(MachineInstOpcode: ARM::VMOVDRR, RC: &ARM::DPRRegClass, Op0, Op1);
3650 }
3651 return 0;
3652}
3653
3654unsigned fastEmit_ARMISD_VMOVDRR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3655 switch (VT.SimpleTy) {
3656 case MVT::i32: return fastEmit_ARMISD_VMOVDRR_MVT_i32_rr(RetVT, Op0, Op1);
3657 default: return 0;
3658 }
3659}
3660
3661// FastEmit functions for ARMISD::VMULLs.
3662
3663unsigned fastEmit_ARMISD_VMULLs_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3664 if (RetVT.SimpleTy != MVT::v8i16)
3665 return 0;
3666 if ((Subtarget->hasNEON())) {
3667 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
3668 }
3669 return 0;
3670}
3671
3672unsigned fastEmit_ARMISD_VMULLs_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3673 if (RetVT.SimpleTy != MVT::v4i32)
3674 return 0;
3675 if ((Subtarget->hasNEON())) {
3676 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
3677 }
3678 return 0;
3679}
3680
3681unsigned fastEmit_ARMISD_VMULLs_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3682 if (RetVT.SimpleTy != MVT::v2i64)
3683 return 0;
3684 if ((Subtarget->hasNEON())) {
3685 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
3686 }
3687 return 0;
3688}
3689
3690unsigned fastEmit_ARMISD_VMULLs_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3691 if (RetVT.SimpleTy != MVT::v2i64)
3692 return 0;
3693 if ((Subtarget->hasMVEIntegerOps())) {
3694 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULLBs32, RC: &ARM::MQPRRegClass, Op0, Op1);
3695 }
3696 return 0;
3697}
3698
3699unsigned fastEmit_ARMISD_VMULLs_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3700 switch (VT.SimpleTy) {
3701 case MVT::v8i8: return fastEmit_ARMISD_VMULLs_MVT_v8i8_rr(RetVT, Op0, Op1);
3702 case MVT::v4i16: return fastEmit_ARMISD_VMULLs_MVT_v4i16_rr(RetVT, Op0, Op1);
3703 case MVT::v2i32: return fastEmit_ARMISD_VMULLs_MVT_v2i32_rr(RetVT, Op0, Op1);
3704 case MVT::v4i32: return fastEmit_ARMISD_VMULLs_MVT_v4i32_rr(RetVT, Op0, Op1);
3705 default: return 0;
3706 }
3707}
3708
3709// FastEmit functions for ARMISD::VMULLu.
3710
3711unsigned fastEmit_ARMISD_VMULLu_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3712 if (RetVT.SimpleTy != MVT::v8i16)
3713 return 0;
3714 if ((Subtarget->hasNEON())) {
3715 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
3716 }
3717 return 0;
3718}
3719
3720unsigned fastEmit_ARMISD_VMULLu_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3721 if (RetVT.SimpleTy != MVT::v4i32)
3722 return 0;
3723 if ((Subtarget->hasNEON())) {
3724 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
3725 }
3726 return 0;
3727}
3728
3729unsigned fastEmit_ARMISD_VMULLu_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3730 if (RetVT.SimpleTy != MVT::v2i64)
3731 return 0;
3732 if ((Subtarget->hasNEON())) {
3733 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
3734 }
3735 return 0;
3736}
3737
3738unsigned fastEmit_ARMISD_VMULLu_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3739 if (RetVT.SimpleTy != MVT::v2i64)
3740 return 0;
3741 if ((Subtarget->hasMVEIntegerOps())) {
3742 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULLBu32, RC: &ARM::MQPRRegClass, Op0, Op1);
3743 }
3744 return 0;
3745}
3746
3747unsigned fastEmit_ARMISD_VMULLu_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3748 switch (VT.SimpleTy) {
3749 case MVT::v8i8: return fastEmit_ARMISD_VMULLu_MVT_v8i8_rr(RetVT, Op0, Op1);
3750 case MVT::v4i16: return fastEmit_ARMISD_VMULLu_MVT_v4i16_rr(RetVT, Op0, Op1);
3751 case MVT::v2i32: return fastEmit_ARMISD_VMULLu_MVT_v2i32_rr(RetVT, Op0, Op1);
3752 case MVT::v4i32: return fastEmit_ARMISD_VMULLu_MVT_v4i32_rr(RetVT, Op0, Op1);
3753 default: return 0;
3754 }
3755}
3756
3757// FastEmit functions for ARMISD::VQDMULH.
3758
3759unsigned fastEmit_ARMISD_VQDMULH_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3760 if (RetVT.SimpleTy != MVT::v16i8)
3761 return 0;
3762 if ((Subtarget->hasMVEIntegerOps())) {
3763 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQDMULHi8, RC: &ARM::MQPRRegClass, Op0, Op1);
3764 }
3765 return 0;
3766}
3767
3768unsigned fastEmit_ARMISD_VQDMULH_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3769 if (RetVT.SimpleTy != MVT::v8i16)
3770 return 0;
3771 if ((Subtarget->hasMVEIntegerOps())) {
3772 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQDMULHi16, RC: &ARM::MQPRRegClass, Op0, Op1);
3773 }
3774 return 0;
3775}
3776
3777unsigned fastEmit_ARMISD_VQDMULH_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3778 if (RetVT.SimpleTy != MVT::v4i32)
3779 return 0;
3780 if ((Subtarget->hasMVEIntegerOps())) {
3781 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQDMULHi32, RC: &ARM::MQPRRegClass, Op0, Op1);
3782 }
3783 return 0;
3784}
3785
3786unsigned fastEmit_ARMISD_VQDMULH_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3787 switch (VT.SimpleTy) {
3788 case MVT::v16i8: return fastEmit_ARMISD_VQDMULH_MVT_v16i8_rr(RetVT, Op0, Op1);
3789 case MVT::v8i16: return fastEmit_ARMISD_VQDMULH_MVT_v8i16_rr(RetVT, Op0, Op1);
3790 case MVT::v4i32: return fastEmit_ARMISD_VQDMULH_MVT_v4i32_rr(RetVT, Op0, Op1);
3791 default: return 0;
3792 }
3793}
3794
3795// FastEmit functions for ARMISD::VSHLs.
3796
3797unsigned fastEmit_ARMISD_VSHLs_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3798 if (RetVT.SimpleTy != MVT::v8i8)
3799 return 0;
3800 if ((Subtarget->hasNEON())) {
3801 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
3802 }
3803 return 0;
3804}
3805
3806unsigned fastEmit_ARMISD_VSHLs_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3807 if (RetVT.SimpleTy != MVT::v16i8)
3808 return 0;
3809 if ((Subtarget->hasMVEIntegerOps())) {
3810 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecs8, RC: &ARM::MQPRRegClass, Op0, Op1);
3811 }
3812 if ((Subtarget->hasNEON())) {
3813 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
3814 }
3815 return 0;
3816}
3817
3818unsigned fastEmit_ARMISD_VSHLs_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3819 if (RetVT.SimpleTy != MVT::v4i16)
3820 return 0;
3821 if ((Subtarget->hasNEON())) {
3822 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
3823 }
3824 return 0;
3825}
3826
3827unsigned fastEmit_ARMISD_VSHLs_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3828 if (RetVT.SimpleTy != MVT::v8i16)
3829 return 0;
3830 if ((Subtarget->hasMVEIntegerOps())) {
3831 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecs16, RC: &ARM::MQPRRegClass, Op0, Op1);
3832 }
3833 if ((Subtarget->hasNEON())) {
3834 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
3835 }
3836 return 0;
3837}
3838
3839unsigned fastEmit_ARMISD_VSHLs_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3840 if (RetVT.SimpleTy != MVT::v2i32)
3841 return 0;
3842 if ((Subtarget->hasNEON())) {
3843 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
3844 }
3845 return 0;
3846}
3847
3848unsigned fastEmit_ARMISD_VSHLs_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3849 if (RetVT.SimpleTy != MVT::v4i32)
3850 return 0;
3851 if ((Subtarget->hasMVEIntegerOps())) {
3852 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecs32, RC: &ARM::MQPRRegClass, Op0, Op1);
3853 }
3854 if ((Subtarget->hasNEON())) {
3855 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
3856 }
3857 return 0;
3858}
3859
3860unsigned fastEmit_ARMISD_VSHLs_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3861 if (RetVT.SimpleTy != MVT::v1i64)
3862 return 0;
3863 if ((Subtarget->hasNEON())) {
3864 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
3865 }
3866 return 0;
3867}
3868
3869unsigned fastEmit_ARMISD_VSHLs_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3870 if (RetVT.SimpleTy != MVT::v2i64)
3871 return 0;
3872 if ((Subtarget->hasNEON())) {
3873 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
3874 }
3875 return 0;
3876}
3877
3878unsigned fastEmit_ARMISD_VSHLs_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3879 switch (VT.SimpleTy) {
3880 case MVT::v8i8: return fastEmit_ARMISD_VSHLs_MVT_v8i8_rr(RetVT, Op0, Op1);
3881 case MVT::v16i8: return fastEmit_ARMISD_VSHLs_MVT_v16i8_rr(RetVT, Op0, Op1);
3882 case MVT::v4i16: return fastEmit_ARMISD_VSHLs_MVT_v4i16_rr(RetVT, Op0, Op1);
3883 case MVT::v8i16: return fastEmit_ARMISD_VSHLs_MVT_v8i16_rr(RetVT, Op0, Op1);
3884 case MVT::v2i32: return fastEmit_ARMISD_VSHLs_MVT_v2i32_rr(RetVT, Op0, Op1);
3885 case MVT::v4i32: return fastEmit_ARMISD_VSHLs_MVT_v4i32_rr(RetVT, Op0, Op1);
3886 case MVT::v1i64: return fastEmit_ARMISD_VSHLs_MVT_v1i64_rr(RetVT, Op0, Op1);
3887 case MVT::v2i64: return fastEmit_ARMISD_VSHLs_MVT_v2i64_rr(RetVT, Op0, Op1);
3888 default: return 0;
3889 }
3890}
3891
3892// FastEmit functions for ARMISD::VSHLu.
3893
3894unsigned fastEmit_ARMISD_VSHLu_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3895 if (RetVT.SimpleTy != MVT::v8i8)
3896 return 0;
3897 if ((Subtarget->hasNEON())) {
3898 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
3899 }
3900 return 0;
3901}
3902
3903unsigned fastEmit_ARMISD_VSHLu_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3904 if (RetVT.SimpleTy != MVT::v16i8)
3905 return 0;
3906 if ((Subtarget->hasMVEIntegerOps())) {
3907 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecu8, RC: &ARM::MQPRRegClass, Op0, Op1);
3908 }
3909 if ((Subtarget->hasNEON())) {
3910 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
3911 }
3912 return 0;
3913}
3914
3915unsigned fastEmit_ARMISD_VSHLu_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3916 if (RetVT.SimpleTy != MVT::v4i16)
3917 return 0;
3918 if ((Subtarget->hasNEON())) {
3919 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
3920 }
3921 return 0;
3922}
3923
3924unsigned fastEmit_ARMISD_VSHLu_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3925 if (RetVT.SimpleTy != MVT::v8i16)
3926 return 0;
3927 if ((Subtarget->hasMVEIntegerOps())) {
3928 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecu16, RC: &ARM::MQPRRegClass, Op0, Op1);
3929 }
3930 if ((Subtarget->hasNEON())) {
3931 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
3932 }
3933 return 0;
3934}
3935
3936unsigned fastEmit_ARMISD_VSHLu_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3937 if (RetVT.SimpleTy != MVT::v2i32)
3938 return 0;
3939 if ((Subtarget->hasNEON())) {
3940 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
3941 }
3942 return 0;
3943}
3944
3945unsigned fastEmit_ARMISD_VSHLu_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3946 if (RetVT.SimpleTy != MVT::v4i32)
3947 return 0;
3948 if ((Subtarget->hasMVEIntegerOps())) {
3949 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecu32, RC: &ARM::MQPRRegClass, Op0, Op1);
3950 }
3951 if ((Subtarget->hasNEON())) {
3952 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
3953 }
3954 return 0;
3955}
3956
3957unsigned fastEmit_ARMISD_VSHLu_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3958 if (RetVT.SimpleTy != MVT::v1i64)
3959 return 0;
3960 if ((Subtarget->hasNEON())) {
3961 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
3962 }
3963 return 0;
3964}
3965
3966unsigned fastEmit_ARMISD_VSHLu_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3967 if (RetVT.SimpleTy != MVT::v2i64)
3968 return 0;
3969 if ((Subtarget->hasNEON())) {
3970 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
3971 }
3972 return 0;
3973}
3974
3975unsigned fastEmit_ARMISD_VSHLu_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
3976 switch (VT.SimpleTy) {
3977 case MVT::v8i8: return fastEmit_ARMISD_VSHLu_MVT_v8i8_rr(RetVT, Op0, Op1);
3978 case MVT::v16i8: return fastEmit_ARMISD_VSHLu_MVT_v16i8_rr(RetVT, Op0, Op1);
3979 case MVT::v4i16: return fastEmit_ARMISD_VSHLu_MVT_v4i16_rr(RetVT, Op0, Op1);
3980 case MVT::v8i16: return fastEmit_ARMISD_VSHLu_MVT_v8i16_rr(RetVT, Op0, Op1);
3981 case MVT::v2i32: return fastEmit_ARMISD_VSHLu_MVT_v2i32_rr(RetVT, Op0, Op1);
3982 case MVT::v4i32: return fastEmit_ARMISD_VSHLu_MVT_v4i32_rr(RetVT, Op0, Op1);
3983 case MVT::v1i64: return fastEmit_ARMISD_VSHLu_MVT_v1i64_rr(RetVT, Op0, Op1);
3984 case MVT::v2i64: return fastEmit_ARMISD_VSHLu_MVT_v2i64_rr(RetVT, Op0, Op1);
3985 default: return 0;
3986 }
3987}
3988
3989// FastEmit functions for ARMISD::VTBL1.
3990
3991unsigned fastEmit_ARMISD_VTBL1_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
3992 if (RetVT.SimpleTy != MVT::v8i8)
3993 return 0;
3994 if ((Subtarget->hasNEON())) {
3995 return fastEmitInst_rr(MachineInstOpcode: ARM::VTBL1, RC: &ARM::DPRRegClass, Op0, Op1);
3996 }
3997 return 0;
3998}
3999
4000unsigned fastEmit_ARMISD_VTBL1_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
4001 switch (VT.SimpleTy) {
4002 case MVT::v8i8: return fastEmit_ARMISD_VTBL1_MVT_v8i8_rr(RetVT, Op0, Op1);
4003 default: return 0;
4004 }
4005}
4006
4007// FastEmit functions for ARMISD::VTST.
4008
4009unsigned fastEmit_ARMISD_VTST_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4010 if (RetVT.SimpleTy != MVT::v8i8)
4011 return 0;
4012 if ((Subtarget->hasNEON())) {
4013 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4014 }
4015 return 0;
4016}
4017
4018unsigned fastEmit_ARMISD_VTST_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4019 if (RetVT.SimpleTy != MVT::v16i8)
4020 return 0;
4021 if ((Subtarget->hasNEON())) {
4022 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4023 }
4024 return 0;
4025}
4026
4027unsigned fastEmit_ARMISD_VTST_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4028 if (RetVT.SimpleTy != MVT::v4i16)
4029 return 0;
4030 if ((Subtarget->hasNEON())) {
4031 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4032 }
4033 return 0;
4034}
4035
4036unsigned fastEmit_ARMISD_VTST_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4037 if (RetVT.SimpleTy != MVT::v8i16)
4038 return 0;
4039 if ((Subtarget->hasNEON())) {
4040 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4041 }
4042 return 0;
4043}
4044
4045unsigned fastEmit_ARMISD_VTST_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4046 if (RetVT.SimpleTy != MVT::v2i32)
4047 return 0;
4048 if ((Subtarget->hasNEON())) {
4049 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4050 }
4051 return 0;
4052}
4053
4054unsigned fastEmit_ARMISD_VTST_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4055 if (RetVT.SimpleTy != MVT::v4i32)
4056 return 0;
4057 if ((Subtarget->hasNEON())) {
4058 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4059 }
4060 return 0;
4061}
4062
4063unsigned fastEmit_ARMISD_VTST_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
4064 switch (VT.SimpleTy) {
4065 case MVT::v8i8: return fastEmit_ARMISD_VTST_MVT_v8i8_rr(RetVT, Op0, Op1);
4066 case MVT::v16i8: return fastEmit_ARMISD_VTST_MVT_v16i8_rr(RetVT, Op0, Op1);
4067 case MVT::v4i16: return fastEmit_ARMISD_VTST_MVT_v4i16_rr(RetVT, Op0, Op1);
4068 case MVT::v8i16: return fastEmit_ARMISD_VTST_MVT_v8i16_rr(RetVT, Op0, Op1);
4069 case MVT::v2i32: return fastEmit_ARMISD_VTST_MVT_v2i32_rr(RetVT, Op0, Op1);
4070 case MVT::v4i32: return fastEmit_ARMISD_VTST_MVT_v4i32_rr(RetVT, Op0, Op1);
4071 default: return 0;
4072 }
4073}
4074
4075// FastEmit functions for ISD::ABDS.
4076
4077unsigned fastEmit_ISD_ABDS_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4078 if (RetVT.SimpleTy != MVT::v8i8)
4079 return 0;
4080 if ((Subtarget->hasNEON())) {
4081 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4082 }
4083 return 0;
4084}
4085
4086unsigned fastEmit_ISD_ABDS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4087 if (RetVT.SimpleTy != MVT::v16i8)
4088 return 0;
4089 if ((Subtarget->hasMVEIntegerOps())) {
4090 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
4091 }
4092 if ((Subtarget->hasNEON())) {
4093 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4094 }
4095 return 0;
4096}
4097
4098unsigned fastEmit_ISD_ABDS_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4099 if (RetVT.SimpleTy != MVT::v4i16)
4100 return 0;
4101 if ((Subtarget->hasNEON())) {
4102 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4103 }
4104 return 0;
4105}
4106
4107unsigned fastEmit_ISD_ABDS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4108 if (RetVT.SimpleTy != MVT::v8i16)
4109 return 0;
4110 if ((Subtarget->hasMVEIntegerOps())) {
4111 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
4112 }
4113 if ((Subtarget->hasNEON())) {
4114 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4115 }
4116 return 0;
4117}
4118
4119unsigned fastEmit_ISD_ABDS_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4120 if (RetVT.SimpleTy != MVT::v2i32)
4121 return 0;
4122 if ((Subtarget->hasNEON())) {
4123 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4124 }
4125 return 0;
4126}
4127
4128unsigned fastEmit_ISD_ABDS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4129 if (RetVT.SimpleTy != MVT::v4i32)
4130 return 0;
4131 if ((Subtarget->hasMVEIntegerOps())) {
4132 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
4133 }
4134 if ((Subtarget->hasNEON())) {
4135 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4136 }
4137 return 0;
4138}
4139
4140unsigned fastEmit_ISD_ABDS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
4141 switch (VT.SimpleTy) {
4142 case MVT::v8i8: return fastEmit_ISD_ABDS_MVT_v8i8_rr(RetVT, Op0, Op1);
4143 case MVT::v16i8: return fastEmit_ISD_ABDS_MVT_v16i8_rr(RetVT, Op0, Op1);
4144 case MVT::v4i16: return fastEmit_ISD_ABDS_MVT_v4i16_rr(RetVT, Op0, Op1);
4145 case MVT::v8i16: return fastEmit_ISD_ABDS_MVT_v8i16_rr(RetVT, Op0, Op1);
4146 case MVT::v2i32: return fastEmit_ISD_ABDS_MVT_v2i32_rr(RetVT, Op0, Op1);
4147 case MVT::v4i32: return fastEmit_ISD_ABDS_MVT_v4i32_rr(RetVT, Op0, Op1);
4148 default: return 0;
4149 }
4150}
4151
4152// FastEmit functions for ISD::ABDU.
4153
4154unsigned fastEmit_ISD_ABDU_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4155 if (RetVT.SimpleTy != MVT::v8i8)
4156 return 0;
4157 if ((Subtarget->hasNEON())) {
4158 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4159 }
4160 return 0;
4161}
4162
4163unsigned fastEmit_ISD_ABDU_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4164 if (RetVT.SimpleTy != MVT::v16i8)
4165 return 0;
4166 if ((Subtarget->hasMVEIntegerOps())) {
4167 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
4168 }
4169 if ((Subtarget->hasNEON())) {
4170 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4171 }
4172 return 0;
4173}
4174
4175unsigned fastEmit_ISD_ABDU_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4176 if (RetVT.SimpleTy != MVT::v4i16)
4177 return 0;
4178 if ((Subtarget->hasNEON())) {
4179 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4180 }
4181 return 0;
4182}
4183
4184unsigned fastEmit_ISD_ABDU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4185 if (RetVT.SimpleTy != MVT::v8i16)
4186 return 0;
4187 if ((Subtarget->hasMVEIntegerOps())) {
4188 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
4189 }
4190 if ((Subtarget->hasNEON())) {
4191 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4192 }
4193 return 0;
4194}
4195
4196unsigned fastEmit_ISD_ABDU_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4197 if (RetVT.SimpleTy != MVT::v2i32)
4198 return 0;
4199 if ((Subtarget->hasNEON())) {
4200 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4201 }
4202 return 0;
4203}
4204
4205unsigned fastEmit_ISD_ABDU_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4206 if (RetVT.SimpleTy != MVT::v4i32)
4207 return 0;
4208 if ((Subtarget->hasMVEIntegerOps())) {
4209 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
4210 }
4211 if ((Subtarget->hasNEON())) {
4212 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4213 }
4214 return 0;
4215}
4216
4217unsigned fastEmit_ISD_ABDU_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
4218 switch (VT.SimpleTy) {
4219 case MVT::v8i8: return fastEmit_ISD_ABDU_MVT_v8i8_rr(RetVT, Op0, Op1);
4220 case MVT::v16i8: return fastEmit_ISD_ABDU_MVT_v16i8_rr(RetVT, Op0, Op1);
4221 case MVT::v4i16: return fastEmit_ISD_ABDU_MVT_v4i16_rr(RetVT, Op0, Op1);
4222 case MVT::v8i16: return fastEmit_ISD_ABDU_MVT_v8i16_rr(RetVT, Op0, Op1);
4223 case MVT::v2i32: return fastEmit_ISD_ABDU_MVT_v2i32_rr(RetVT, Op0, Op1);
4224 case MVT::v4i32: return fastEmit_ISD_ABDU_MVT_v4i32_rr(RetVT, Op0, Op1);
4225 default: return 0;
4226 }
4227}
4228
4229// FastEmit functions for ISD::ADD.
4230
4231unsigned fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4232 if (RetVT.SimpleTy != MVT::i32)
4233 return 0;
4234 if ((Subtarget->isThumb2())) {
4235 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ADDrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4236 }
4237 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
4238 return fastEmitInst_rr(MachineInstOpcode: ARM::tADDrr, RC: &ARM::tGPRRegClass, Op0, Op1);
4239 }
4240 if ((!Subtarget->isThumb())) {
4241 return fastEmitInst_rr(MachineInstOpcode: ARM::ADDrr, RC: &ARM::GPRRegClass, Op0, Op1);
4242 }
4243 return 0;
4244}
4245
4246unsigned fastEmit_ISD_ADD_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4247 if (RetVT.SimpleTy != MVT::v8i8)
4248 return 0;
4249 if ((Subtarget->hasNEON())) {
4250 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4251 }
4252 return 0;
4253}
4254
4255unsigned fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4256 if (RetVT.SimpleTy != MVT::v16i8)
4257 return 0;
4258 if ((Subtarget->hasMVEIntegerOps())) {
4259 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDi8, RC: &ARM::MQPRRegClass, Op0, Op1);
4260 }
4261 if ((Subtarget->hasNEON())) {
4262 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4263 }
4264 return 0;
4265}
4266
4267unsigned fastEmit_ISD_ADD_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4268 if (RetVT.SimpleTy != MVT::v4i16)
4269 return 0;
4270 if ((Subtarget->hasNEON())) {
4271 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4272 }
4273 return 0;
4274}
4275
4276unsigned fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4277 if (RetVT.SimpleTy != MVT::v8i16)
4278 return 0;
4279 if ((Subtarget->hasMVEIntegerOps())) {
4280 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDi16, RC: &ARM::MQPRRegClass, Op0, Op1);
4281 }
4282 if ((Subtarget->hasNEON())) {
4283 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4284 }
4285 return 0;
4286}
4287
4288unsigned fastEmit_ISD_ADD_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4289 if (RetVT.SimpleTy != MVT::v2i32)
4290 return 0;
4291 if ((Subtarget->hasNEON())) {
4292 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4293 }
4294 return 0;
4295}
4296
4297unsigned fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4298 if (RetVT.SimpleTy != MVT::v4i32)
4299 return 0;
4300 if ((Subtarget->hasMVEIntegerOps())) {
4301 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDi32, RC: &ARM::MQPRRegClass, Op0, Op1);
4302 }
4303 if ((Subtarget->hasNEON())) {
4304 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4305 }
4306 return 0;
4307}
4308
4309unsigned fastEmit_ISD_ADD_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4310 if (RetVT.SimpleTy != MVT::v1i64)
4311 return 0;
4312 if ((Subtarget->hasNEON())) {
4313 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
4314 }
4315 return 0;
4316}
4317
4318unsigned fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4319 if (RetVT.SimpleTy != MVT::v2i64)
4320 return 0;
4321 if ((Subtarget->hasNEON())) {
4322 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4323 }
4324 return 0;
4325}
4326
4327unsigned fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
4328 switch (VT.SimpleTy) {
4329 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
4330 case MVT::v8i8: return fastEmit_ISD_ADD_MVT_v8i8_rr(RetVT, Op0, Op1);
4331 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
4332 case MVT::v4i16: return fastEmit_ISD_ADD_MVT_v4i16_rr(RetVT, Op0, Op1);
4333 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
4334 case MVT::v2i32: return fastEmit_ISD_ADD_MVT_v2i32_rr(RetVT, Op0, Op1);
4335 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
4336 case MVT::v1i64: return fastEmit_ISD_ADD_MVT_v1i64_rr(RetVT, Op0, Op1);
4337 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
4338 default: return 0;
4339 }
4340}
4341
4342// FastEmit functions for ISD::AND.
4343
4344unsigned fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4345 if (RetVT.SimpleTy != MVT::i32)
4346 return 0;
4347 if ((Subtarget->isThumb2())) {
4348 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ANDrr, RC: &ARM::rGPRRegClass, Op0, Op1);
4349 }
4350 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
4351 return fastEmitInst_rr(MachineInstOpcode: ARM::tAND, RC: &ARM::tGPRRegClass, Op0, Op1);
4352 }
4353 if ((!Subtarget->isThumb())) {
4354 return fastEmitInst_rr(MachineInstOpcode: ARM::ANDrr, RC: &ARM::GPRRegClass, Op0, Op1);
4355 }
4356 return 0;
4357}
4358
4359unsigned fastEmit_ISD_AND_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4360 if (RetVT.SimpleTy != MVT::v8i8)
4361 return 0;
4362 if ((Subtarget->hasNEON())) {
4363 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
4364 }
4365 return 0;
4366}
4367
4368unsigned fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4369 if (RetVT.SimpleTy != MVT::v16i8)
4370 return 0;
4371 if ((Subtarget->hasMVEIntegerOps())) {
4372 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
4373 }
4374 if ((Subtarget->hasNEON())) {
4375 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
4376 }
4377 return 0;
4378}
4379
4380unsigned fastEmit_ISD_AND_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4381 if (RetVT.SimpleTy != MVT::v4i16)
4382 return 0;
4383 if ((Subtarget->hasNEON())) {
4384 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
4385 }
4386 return 0;
4387}
4388
4389unsigned fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4390 if (RetVT.SimpleTy != MVT::v8i16)
4391 return 0;
4392 if ((Subtarget->hasMVEIntegerOps())) {
4393 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
4394 }
4395 if ((Subtarget->hasNEON())) {
4396 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
4397 }
4398 return 0;
4399}
4400
4401unsigned fastEmit_ISD_AND_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4402 if (RetVT.SimpleTy != MVT::v2i32)
4403 return 0;
4404 if ((Subtarget->hasNEON())) {
4405 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
4406 }
4407 return 0;
4408}
4409
4410unsigned fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4411 if (RetVT.SimpleTy != MVT::v4i32)
4412 return 0;
4413 if ((Subtarget->hasMVEIntegerOps())) {
4414 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
4415 }
4416 if ((Subtarget->hasNEON())) {
4417 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
4418 }
4419 return 0;
4420}
4421
4422unsigned fastEmit_ISD_AND_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4423 if (RetVT.SimpleTy != MVT::v1i64)
4424 return 0;
4425 if ((Subtarget->hasNEON())) {
4426 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
4427 }
4428 return 0;
4429}
4430
4431unsigned fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4432 if (RetVT.SimpleTy != MVT::v2i64)
4433 return 0;
4434 if ((Subtarget->hasMVEIntegerOps())) {
4435 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
4436 }
4437 if ((Subtarget->hasNEON())) {
4438 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
4439 }
4440 return 0;
4441}
4442
4443unsigned fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
4444 switch (VT.SimpleTy) {
4445 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
4446 case MVT::v8i8: return fastEmit_ISD_AND_MVT_v8i8_rr(RetVT, Op0, Op1);
4447 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
4448 case MVT::v4i16: return fastEmit_ISD_AND_MVT_v4i16_rr(RetVT, Op0, Op1);
4449 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
4450 case MVT::v2i32: return fastEmit_ISD_AND_MVT_v2i32_rr(RetVT, Op0, Op1);
4451 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
4452 case MVT::v1i64: return fastEmit_ISD_AND_MVT_v1i64_rr(RetVT, Op0, Op1);
4453 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
4454 default: return 0;
4455 }
4456}
4457
4458// FastEmit functions for ISD::AVGCEILS.
4459
4460unsigned fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4461 if (RetVT.SimpleTy != MVT::v16i8)
4462 return 0;
4463 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
4464}
4465
4466unsigned fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4467 if (RetVT.SimpleTy != MVT::v8i16)
4468 return 0;
4469 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
4470}
4471
4472unsigned fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4473 if (RetVT.SimpleTy != MVT::v4i32)
4474 return 0;
4475 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
4476}
4477
4478unsigned fastEmit_ISD_AVGCEILS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
4479 switch (VT.SimpleTy) {
4480 case MVT::v16i8: return fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(RetVT, Op0, Op1);
4481 case MVT::v8i16: return fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(RetVT, Op0, Op1);
4482 case MVT::v4i32: return fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(RetVT, Op0, Op1);
4483 default: return 0;
4484 }
4485}
4486
4487// FastEmit functions for ISD::AVGCEILU.
4488
4489unsigned fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4490 if (RetVT.SimpleTy != MVT::v16i8)
4491 return 0;
4492 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
4493}
4494
4495unsigned fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4496 if (RetVT.SimpleTy != MVT::v8i16)
4497 return 0;
4498 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
4499}
4500
4501unsigned fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4502 if (RetVT.SimpleTy != MVT::v4i32)
4503 return 0;
4504 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
4505}
4506
4507unsigned fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
4508 switch (VT.SimpleTy) {
4509 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
4510 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
4511 case MVT::v4i32: return fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(RetVT, Op0, Op1);
4512 default: return 0;
4513 }
4514}
4515
4516// FastEmit functions for ISD::AVGFLOORS.
4517
4518unsigned fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4519 if (RetVT.SimpleTy != MVT::v16i8)
4520 return 0;
4521 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
4522}
4523
4524unsigned fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4525 if (RetVT.SimpleTy != MVT::v8i16)
4526 return 0;
4527 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
4528}
4529
4530unsigned fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4531 if (RetVT.SimpleTy != MVT::v4i32)
4532 return 0;
4533 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
4534}
4535
4536unsigned fastEmit_ISD_AVGFLOORS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
4537 switch (VT.SimpleTy) {
4538 case MVT::v16i8: return fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(RetVT, Op0, Op1);
4539 case MVT::v8i16: return fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(RetVT, Op0, Op1);
4540 case MVT::v4i32: return fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(RetVT, Op0, Op1);
4541 default: return 0;
4542 }
4543}
4544
4545// FastEmit functions for ISD::AVGFLOORU.
4546
4547unsigned fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4548 if (RetVT.SimpleTy != MVT::v16i8)
4549 return 0;
4550 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
4551}
4552
4553unsigned fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4554 if (RetVT.SimpleTy != MVT::v8i16)
4555 return 0;
4556 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
4557}
4558
4559unsigned fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4560 if (RetVT.SimpleTy != MVT::v4i32)
4561 return 0;
4562 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
4563}
4564
4565unsigned fastEmit_ISD_AVGFLOORU_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
4566 switch (VT.SimpleTy) {
4567 case MVT::v16i8: return fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(RetVT, Op0, Op1);
4568 case MVT::v8i16: return fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(RetVT, Op0, Op1);
4569 case MVT::v4i32: return fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(RetVT, Op0, Op1);
4570 default: return 0;
4571 }
4572}
4573
4574// FastEmit functions for ISD::FADD.
4575
4576unsigned fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4577 if (RetVT.SimpleTy != MVT::f16)
4578 return 0;
4579 if ((Subtarget->hasFullFP16())) {
4580 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDH, RC: &ARM::HPRRegClass, Op0, Op1);
4581 }
4582 return 0;
4583}
4584
4585unsigned fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4586 if (RetVT.SimpleTy != MVT::f32)
4587 return 0;
4588 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
4589 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDS, RC: &ARM::SPRRegClass, Op0, Op1);
4590 }
4591 return 0;
4592}
4593
4594unsigned fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4595 if (RetVT.SimpleTy != MVT::f64)
4596 return 0;
4597 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
4598 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDD, RC: &ARM::DPRRegClass, Op0, Op1);
4599 }
4600 return 0;
4601}
4602
4603unsigned fastEmit_ISD_FADD_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4604 if (RetVT.SimpleTy != MVT::v4f16)
4605 return 0;
4606 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4607 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDhd, RC: &ARM::DPRRegClass, Op0, Op1);
4608 }
4609 return 0;
4610}
4611
4612unsigned fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4613 if (RetVT.SimpleTy != MVT::v8f16)
4614 return 0;
4615 if ((Subtarget->hasMVEFloatOps())) {
4616 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDf16, RC: &ARM::MQPRRegClass, Op0, Op1);
4617 }
4618 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4619 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDhq, RC: &ARM::QPRRegClass, Op0, Op1);
4620 }
4621 return 0;
4622}
4623
4624unsigned fastEmit_ISD_FADD_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4625 if (RetVT.SimpleTy != MVT::v2f32)
4626 return 0;
4627 if ((Subtarget->hasNEON())) {
4628 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDfd, RC: &ARM::DPRRegClass, Op0, Op1);
4629 }
4630 return 0;
4631}
4632
4633unsigned fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4634 if (RetVT.SimpleTy != MVT::v4f32)
4635 return 0;
4636 if ((Subtarget->hasMVEFloatOps())) {
4637 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDf32, RC: &ARM::MQPRRegClass, Op0, Op1);
4638 }
4639 if ((Subtarget->hasNEON())) {
4640 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDfq, RC: &ARM::QPRRegClass, Op0, Op1);
4641 }
4642 return 0;
4643}
4644
4645unsigned fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
4646 switch (VT.SimpleTy) {
4647 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
4648 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
4649 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
4650 case MVT::v4f16: return fastEmit_ISD_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
4651 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
4652 case MVT::v2f32: return fastEmit_ISD_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
4653 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
4654 default: return 0;
4655 }
4656}
4657
4658// FastEmit functions for ISD::FDIV.
4659
4660unsigned fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4661 if (RetVT.SimpleTy != MVT::f16)
4662 return 0;
4663 if ((Subtarget->hasFullFP16())) {
4664 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVH, RC: &ARM::HPRRegClass, Op0, Op1);
4665 }
4666 return 0;
4667}
4668
4669unsigned fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4670 if (RetVT.SimpleTy != MVT::f32)
4671 return 0;
4672 if ((Subtarget->hasVFP2Base())) {
4673 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVS, RC: &ARM::SPRRegClass, Op0, Op1);
4674 }
4675 return 0;
4676}
4677
4678unsigned fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4679 if (RetVT.SimpleTy != MVT::f64)
4680 return 0;
4681 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
4682 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVD, RC: &ARM::DPRRegClass, Op0, Op1);
4683 }
4684 return 0;
4685}
4686
4687unsigned fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
4688 switch (VT.SimpleTy) {
4689 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
4690 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
4691 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
4692 default: return 0;
4693 }
4694}
4695
4696// FastEmit functions for ISD::FMAXIMUM.
4697
4698unsigned fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4699 if (RetVT.SimpleTy != MVT::v4f16)
4700 return 0;
4701 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4702 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXhd, RC: &ARM::DPRRegClass, Op0, Op1);
4703 }
4704 return 0;
4705}
4706
4707unsigned fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4708 if (RetVT.SimpleTy != MVT::v8f16)
4709 return 0;
4710 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4711 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXhq, RC: &ARM::QPRRegClass, Op0, Op1);
4712 }
4713 return 0;
4714}
4715
4716unsigned fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4717 if (RetVT.SimpleTy != MVT::v2f32)
4718 return 0;
4719 if ((Subtarget->hasNEON())) {
4720 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXfd, RC: &ARM::DPRRegClass, Op0, Op1);
4721 }
4722 return 0;
4723}
4724
4725unsigned fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4726 if (RetVT.SimpleTy != MVT::v4f32)
4727 return 0;
4728 if ((Subtarget->hasNEON())) {
4729 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXfq, RC: &ARM::QPRRegClass, Op0, Op1);
4730 }
4731 return 0;
4732}
4733
4734unsigned fastEmit_ISD_FMAXIMUM_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
4735 switch (VT.SimpleTy) {
4736 case MVT::v4f16: return fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
4737 case MVT::v8f16: return fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
4738 case MVT::v2f32: return fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
4739 case MVT::v4f32: return fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
4740 default: return 0;
4741 }
4742}
4743
4744// FastEmit functions for ISD::FMAXNUM.
4745
4746unsigned fastEmit_ISD_FMAXNUM_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4747 if (RetVT.SimpleTy != MVT::f16)
4748 return 0;
4749 if ((Subtarget->hasFullFP16())) {
4750 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMH, RC: &ARM::HPRRegClass, Op0, Op1);
4751 }
4752 return 0;
4753}
4754
4755unsigned fastEmit_ISD_FMAXNUM_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4756 if (RetVT.SimpleTy != MVT::f32)
4757 return 0;
4758 if ((Subtarget->hasFPARMv8Base())) {
4759 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMS, RC: &ARM::SPRRegClass, Op0, Op1);
4760 }
4761 return 0;
4762}
4763
4764unsigned fastEmit_ISD_FMAXNUM_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4765 if (RetVT.SimpleTy != MVT::f64)
4766 return 0;
4767 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
4768 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMD, RC: &ARM::DPRRegClass, Op0, Op1);
4769 }
4770 return 0;
4771}
4772
4773unsigned fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4774 if (RetVT.SimpleTy != MVT::v4f16)
4775 return 0;
4776 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4777 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNDh, RC: &ARM::DPRRegClass, Op0, Op1);
4778 }
4779 return 0;
4780}
4781
4782unsigned fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4783 if (RetVT.SimpleTy != MVT::v8f16)
4784 return 0;
4785 if ((Subtarget->hasMVEFloatOps())) {
4786 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXNMf16, RC: &ARM::MQPRRegClass, Op0, Op1);
4787 }
4788 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4789 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNQh, RC: &ARM::QPRRegClass, Op0, Op1);
4790 }
4791 return 0;
4792}
4793
4794unsigned fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4795 if (RetVT.SimpleTy != MVT::v2f32)
4796 return 0;
4797 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
4798 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNDf, RC: &ARM::DPRRegClass, Op0, Op1);
4799 }
4800 return 0;
4801}
4802
4803unsigned fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4804 if (RetVT.SimpleTy != MVT::v4f32)
4805 return 0;
4806 if ((Subtarget->hasMVEFloatOps())) {
4807 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXNMf32, RC: &ARM::MQPRRegClass, Op0, Op1);
4808 }
4809 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
4810 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNQf, RC: &ARM::QPRRegClass, Op0, Op1);
4811 }
4812 return 0;
4813}
4814
4815unsigned fastEmit_ISD_FMAXNUM_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
4816 switch (VT.SimpleTy) {
4817 case MVT::f16: return fastEmit_ISD_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
4818 case MVT::f32: return fastEmit_ISD_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
4819 case MVT::f64: return fastEmit_ISD_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
4820 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
4821 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
4822 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
4823 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
4824 default: return 0;
4825 }
4826}
4827
4828// FastEmit functions for ISD::FMINIMUM.
4829
4830unsigned fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4831 if (RetVT.SimpleTy != MVT::v4f16)
4832 return 0;
4833 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4834 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINhd, RC: &ARM::DPRRegClass, Op0, Op1);
4835 }
4836 return 0;
4837}
4838
4839unsigned fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4840 if (RetVT.SimpleTy != MVT::v8f16)
4841 return 0;
4842 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4843 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINhq, RC: &ARM::QPRRegClass, Op0, Op1);
4844 }
4845 return 0;
4846}
4847
4848unsigned fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4849 if (RetVT.SimpleTy != MVT::v2f32)
4850 return 0;
4851 if ((Subtarget->hasNEON())) {
4852 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINfd, RC: &ARM::DPRRegClass, Op0, Op1);
4853 }
4854 return 0;
4855}
4856
4857unsigned fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4858 if (RetVT.SimpleTy != MVT::v4f32)
4859 return 0;
4860 if ((Subtarget->hasNEON())) {
4861 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINfq, RC: &ARM::QPRRegClass, Op0, Op1);
4862 }
4863 return 0;
4864}
4865
4866unsigned fastEmit_ISD_FMINIMUM_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
4867 switch (VT.SimpleTy) {
4868 case MVT::v4f16: return fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
4869 case MVT::v8f16: return fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
4870 case MVT::v2f32: return fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
4871 case MVT::v4f32: return fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
4872 default: return 0;
4873 }
4874}
4875
4876// FastEmit functions for ISD::FMINNUM.
4877
4878unsigned fastEmit_ISD_FMINNUM_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4879 if (RetVT.SimpleTy != MVT::f16)
4880 return 0;
4881 if ((Subtarget->hasFullFP16())) {
4882 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMH, RC: &ARM::HPRRegClass, Op0, Op1);
4883 }
4884 return 0;
4885}
4886
4887unsigned fastEmit_ISD_FMINNUM_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4888 if (RetVT.SimpleTy != MVT::f32)
4889 return 0;
4890 if ((Subtarget->hasFPARMv8Base())) {
4891 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMS, RC: &ARM::SPRRegClass, Op0, Op1);
4892 }
4893 return 0;
4894}
4895
4896unsigned fastEmit_ISD_FMINNUM_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4897 if (RetVT.SimpleTy != MVT::f64)
4898 return 0;
4899 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
4900 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMD, RC: &ARM::DPRRegClass, Op0, Op1);
4901 }
4902 return 0;
4903}
4904
4905unsigned fastEmit_ISD_FMINNUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4906 if (RetVT.SimpleTy != MVT::v4f16)
4907 return 0;
4908 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4909 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNDh, RC: &ARM::DPRRegClass, Op0, Op1);
4910 }
4911 return 0;
4912}
4913
4914unsigned fastEmit_ISD_FMINNUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4915 if (RetVT.SimpleTy != MVT::v8f16)
4916 return 0;
4917 if ((Subtarget->hasMVEFloatOps())) {
4918 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINNMf16, RC: &ARM::MQPRRegClass, Op0, Op1);
4919 }
4920 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4921 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNQh, RC: &ARM::QPRRegClass, Op0, Op1);
4922 }
4923 return 0;
4924}
4925
4926unsigned fastEmit_ISD_FMINNUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4927 if (RetVT.SimpleTy != MVT::v2f32)
4928 return 0;
4929 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
4930 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNDf, RC: &ARM::DPRRegClass, Op0, Op1);
4931 }
4932 return 0;
4933}
4934
4935unsigned fastEmit_ISD_FMINNUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4936 if (RetVT.SimpleTy != MVT::v4f32)
4937 return 0;
4938 if ((Subtarget->hasMVEFloatOps())) {
4939 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINNMf32, RC: &ARM::MQPRRegClass, Op0, Op1);
4940 }
4941 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
4942 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNQf, RC: &ARM::QPRRegClass, Op0, Op1);
4943 }
4944 return 0;
4945}
4946
4947unsigned fastEmit_ISD_FMINNUM_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
4948 switch (VT.SimpleTy) {
4949 case MVT::f16: return fastEmit_ISD_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
4950 case MVT::f32: return fastEmit_ISD_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
4951 case MVT::f64: return fastEmit_ISD_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
4952 case MVT::v4f16: return fastEmit_ISD_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
4953 case MVT::v8f16: return fastEmit_ISD_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
4954 case MVT::v2f32: return fastEmit_ISD_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
4955 case MVT::v4f32: return fastEmit_ISD_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
4956 default: return 0;
4957 }
4958}
4959
4960// FastEmit functions for ISD::FMUL.
4961
4962unsigned fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4963 if (RetVT.SimpleTy != MVT::f16)
4964 return 0;
4965 if ((Subtarget->hasFullFP16())) {
4966 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULH, RC: &ARM::HPRRegClass, Op0, Op1);
4967 }
4968 return 0;
4969}
4970
4971unsigned fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4972 if (RetVT.SimpleTy != MVT::f32)
4973 return 0;
4974 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
4975 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULS, RC: &ARM::SPRRegClass, Op0, Op1);
4976 }
4977 return 0;
4978}
4979
4980unsigned fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4981 if (RetVT.SimpleTy != MVT::f64)
4982 return 0;
4983 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
4984 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULD, RC: &ARM::DPRRegClass, Op0, Op1);
4985 }
4986 return 0;
4987}
4988
4989unsigned fastEmit_ISD_FMUL_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4990 if (RetVT.SimpleTy != MVT::v4f16)
4991 return 0;
4992 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4993 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULhd, RC: &ARM::DPRRegClass, Op0, Op1);
4994 }
4995 return 0;
4996}
4997
4998unsigned fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
4999 if (RetVT.SimpleTy != MVT::v8f16)
5000 return 0;
5001 if ((Subtarget->hasMVEFloatOps())) {
5002 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5003 }
5004 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5005 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULhq, RC: &ARM::QPRRegClass, Op0, Op1);
5006 }
5007 return 0;
5008}
5009
5010unsigned fastEmit_ISD_FMUL_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5011 if (RetVT.SimpleTy != MVT::v2f32)
5012 return 0;
5013 if ((Subtarget->hasNEON())) {
5014 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULfd, RC: &ARM::DPRRegClass, Op0, Op1);
5015 }
5016 return 0;
5017}
5018
5019unsigned fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5020 if (RetVT.SimpleTy != MVT::v4f32)
5021 return 0;
5022 if ((Subtarget->hasMVEFloatOps())) {
5023 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5024 }
5025 if ((Subtarget->hasNEON())) {
5026 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULfq, RC: &ARM::QPRRegClass, Op0, Op1);
5027 }
5028 return 0;
5029}
5030
5031unsigned fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
5032 switch (VT.SimpleTy) {
5033 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
5034 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
5035 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
5036 case MVT::v4f16: return fastEmit_ISD_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
5037 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
5038 case MVT::v2f32: return fastEmit_ISD_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
5039 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
5040 default: return 0;
5041 }
5042}
5043
5044// FastEmit functions for ISD::FSUB.
5045
5046unsigned fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5047 if (RetVT.SimpleTy != MVT::f16)
5048 return 0;
5049 if ((Subtarget->hasFullFP16())) {
5050 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBH, RC: &ARM::HPRRegClass, Op0, Op1);
5051 }
5052 return 0;
5053}
5054
5055unsigned fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5056 if (RetVT.SimpleTy != MVT::f32)
5057 return 0;
5058 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
5059 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBS, RC: &ARM::SPRRegClass, Op0, Op1);
5060 }
5061 return 0;
5062}
5063
5064unsigned fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5065 if (RetVT.SimpleTy != MVT::f64)
5066 return 0;
5067 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
5068 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBD, RC: &ARM::DPRRegClass, Op0, Op1);
5069 }
5070 return 0;
5071}
5072
5073unsigned fastEmit_ISD_FSUB_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5074 if (RetVT.SimpleTy != MVT::v4f16)
5075 return 0;
5076 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5077 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBhd, RC: &ARM::DPRRegClass, Op0, Op1);
5078 }
5079 return 0;
5080}
5081
5082unsigned fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5083 if (RetVT.SimpleTy != MVT::v8f16)
5084 return 0;
5085 if ((Subtarget->hasMVEFloatOps())) {
5086 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5087 }
5088 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5089 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBhq, RC: &ARM::QPRRegClass, Op0, Op1);
5090 }
5091 return 0;
5092}
5093
5094unsigned fastEmit_ISD_FSUB_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5095 if (RetVT.SimpleTy != MVT::v2f32)
5096 return 0;
5097 if ((Subtarget->hasNEON())) {
5098 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBfd, RC: &ARM::DPRRegClass, Op0, Op1);
5099 }
5100 return 0;
5101}
5102
5103unsigned fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5104 if (RetVT.SimpleTy != MVT::v4f32)
5105 return 0;
5106 if ((Subtarget->hasMVEFloatOps())) {
5107 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5108 }
5109 if ((Subtarget->hasNEON())) {
5110 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBfq, RC: &ARM::QPRRegClass, Op0, Op1);
5111 }
5112 return 0;
5113}
5114
5115unsigned fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
5116 switch (VT.SimpleTy) {
5117 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
5118 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
5119 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
5120 case MVT::v4f16: return fastEmit_ISD_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
5121 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
5122 case MVT::v2f32: return fastEmit_ISD_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
5123 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
5124 default: return 0;
5125 }
5126}
5127
5128// FastEmit functions for ISD::MUL.
5129
5130unsigned fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5131 if (RetVT.SimpleTy != MVT::i32)
5132 return 0;
5133 if ((Subtarget->isThumb2())) {
5134 return fastEmitInst_rr(MachineInstOpcode: ARM::t2MUL, RC: &ARM::rGPRRegClass, Op0, Op1);
5135 }
5136 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5137 return fastEmitInst_rr(MachineInstOpcode: ARM::tMUL, RC: &ARM::tGPRRegClass, Op0, Op1);
5138 }
5139 if ((!Subtarget->isThumb()) && (!Subtarget->hasV6Ops()) && (Subtarget->useMulOps())) {
5140 return fastEmitInst_rr(MachineInstOpcode: ARM::MULv5, RC: &ARM::GPRnopcRegClass, Op0, Op1);
5141 }
5142 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
5143 return fastEmitInst_rr(MachineInstOpcode: ARM::MUL, RC: &ARM::GPRnopcRegClass, Op0, Op1);
5144 }
5145 return 0;
5146}
5147
5148unsigned fastEmit_ISD_MUL_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5149 if (RetVT.SimpleTy != MVT::v8i8)
5150 return 0;
5151 if ((Subtarget->hasNEON())) {
5152 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
5153 }
5154 return 0;
5155}
5156
5157unsigned fastEmit_ISD_MUL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5158 if (RetVT.SimpleTy != MVT::v16i8)
5159 return 0;
5160 if ((Subtarget->hasMVEIntegerOps())) {
5161 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULi8, RC: &ARM::MQPRRegClass, Op0, Op1);
5162 }
5163 if ((Subtarget->hasNEON())) {
5164 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
5165 }
5166 return 0;
5167}
5168
5169unsigned fastEmit_ISD_MUL_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5170 if (RetVT.SimpleTy != MVT::v4i16)
5171 return 0;
5172 if ((Subtarget->hasNEON())) {
5173 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
5174 }
5175 return 0;
5176}
5177
5178unsigned fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5179 if (RetVT.SimpleTy != MVT::v8i16)
5180 return 0;
5181 if ((Subtarget->hasMVEIntegerOps())) {
5182 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULi16, RC: &ARM::MQPRRegClass, Op0, Op1);
5183 }
5184 if ((Subtarget->hasNEON())) {
5185 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
5186 }
5187 return 0;
5188}
5189
5190unsigned fastEmit_ISD_MUL_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5191 if (RetVT.SimpleTy != MVT::v2i32)
5192 return 0;
5193 if ((Subtarget->hasNEON())) {
5194 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
5195 }
5196 return 0;
5197}
5198
5199unsigned fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5200 if (RetVT.SimpleTy != MVT::v4i32)
5201 return 0;
5202 if ((Subtarget->hasMVEIntegerOps())) {
5203 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULi32, RC: &ARM::MQPRRegClass, Op0, Op1);
5204 }
5205 if ((Subtarget->hasNEON())) {
5206 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
5207 }
5208 return 0;
5209}
5210
5211unsigned fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
5212 switch (VT.SimpleTy) {
5213 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op1);
5214 case MVT::v8i8: return fastEmit_ISD_MUL_MVT_v8i8_rr(RetVT, Op0, Op1);
5215 case MVT::v16i8: return fastEmit_ISD_MUL_MVT_v16i8_rr(RetVT, Op0, Op1);
5216 case MVT::v4i16: return fastEmit_ISD_MUL_MVT_v4i16_rr(RetVT, Op0, Op1);
5217 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
5218 case MVT::v2i32: return fastEmit_ISD_MUL_MVT_v2i32_rr(RetVT, Op0, Op1);
5219 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
5220 default: return 0;
5221 }
5222}
5223
5224// FastEmit functions for ISD::MULHS.
5225
5226unsigned fastEmit_ISD_MULHS_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5227 if (RetVT.SimpleTy != MVT::i32)
5228 return 0;
5229 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
5230 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SMMUL, RC: &ARM::rGPRRegClass, Op0, Op1);
5231 }
5232 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
5233 return fastEmitInst_rr(MachineInstOpcode: ARM::SMMUL, RC: &ARM::GPRRegClass, Op0, Op1);
5234 }
5235 return 0;
5236}
5237
5238unsigned fastEmit_ISD_MULHS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5239 if (RetVT.SimpleTy != MVT::v16i8)
5240 return 0;
5241 if ((Subtarget->hasMVEIntegerOps())) {
5242 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHs8, RC: &ARM::MQPRRegClass, Op0, Op1);
5243 }
5244 return 0;
5245}
5246
5247unsigned fastEmit_ISD_MULHS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5248 if (RetVT.SimpleTy != MVT::v8i16)
5249 return 0;
5250 if ((Subtarget->hasMVEIntegerOps())) {
5251 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHs16, RC: &ARM::MQPRRegClass, Op0, Op1);
5252 }
5253 return 0;
5254}
5255
5256unsigned fastEmit_ISD_MULHS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5257 if (RetVT.SimpleTy != MVT::v4i32)
5258 return 0;
5259 if ((Subtarget->hasMVEIntegerOps())) {
5260 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHs32, RC: &ARM::MQPRRegClass, Op0, Op1);
5261 }
5262 return 0;
5263}
5264
5265unsigned fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
5266 switch (VT.SimpleTy) {
5267 case MVT::i32: return fastEmit_ISD_MULHS_MVT_i32_rr(RetVT, Op0, Op1);
5268 case MVT::v16i8: return fastEmit_ISD_MULHS_MVT_v16i8_rr(RetVT, Op0, Op1);
5269 case MVT::v8i16: return fastEmit_ISD_MULHS_MVT_v8i16_rr(RetVT, Op0, Op1);
5270 case MVT::v4i32: return fastEmit_ISD_MULHS_MVT_v4i32_rr(RetVT, Op0, Op1);
5271 default: return 0;
5272 }
5273}
5274
5275// FastEmit functions for ISD::MULHU.
5276
5277unsigned fastEmit_ISD_MULHU_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5278 if (RetVT.SimpleTy != MVT::v16i8)
5279 return 0;
5280 if ((Subtarget->hasMVEIntegerOps())) {
5281 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHu8, RC: &ARM::MQPRRegClass, Op0, Op1);
5282 }
5283 return 0;
5284}
5285
5286unsigned fastEmit_ISD_MULHU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5287 if (RetVT.SimpleTy != MVT::v8i16)
5288 return 0;
5289 if ((Subtarget->hasMVEIntegerOps())) {
5290 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHu16, RC: &ARM::MQPRRegClass, Op0, Op1);
5291 }
5292 return 0;
5293}
5294
5295unsigned fastEmit_ISD_MULHU_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5296 if (RetVT.SimpleTy != MVT::v4i32)
5297 return 0;
5298 if ((Subtarget->hasMVEIntegerOps())) {
5299 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHu32, RC: &ARM::MQPRRegClass, Op0, Op1);
5300 }
5301 return 0;
5302}
5303
5304unsigned fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
5305 switch (VT.SimpleTy) {
5306 case MVT::v16i8: return fastEmit_ISD_MULHU_MVT_v16i8_rr(RetVT, Op0, Op1);
5307 case MVT::v8i16: return fastEmit_ISD_MULHU_MVT_v8i16_rr(RetVT, Op0, Op1);
5308 case MVT::v4i32: return fastEmit_ISD_MULHU_MVT_v4i32_rr(RetVT, Op0, Op1);
5309 default: return 0;
5310 }
5311}
5312
5313// FastEmit functions for ISD::OR.
5314
5315unsigned fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5316 if (RetVT.SimpleTy != MVT::i32)
5317 return 0;
5318 if ((Subtarget->isThumb2())) {
5319 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ORRrr, RC: &ARM::rGPRRegClass, Op0, Op1);
5320 }
5321 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5322 return fastEmitInst_rr(MachineInstOpcode: ARM::tORR, RC: &ARM::tGPRRegClass, Op0, Op1);
5323 }
5324 if ((!Subtarget->isThumb())) {
5325 return fastEmitInst_rr(MachineInstOpcode: ARM::ORRrr, RC: &ARM::GPRRegClass, Op0, Op1);
5326 }
5327 return 0;
5328}
5329
5330unsigned fastEmit_ISD_OR_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5331 if (RetVT.SimpleTy != MVT::v8i8)
5332 return 0;
5333 if ((Subtarget->hasNEON())) {
5334 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
5335 }
5336 return 0;
5337}
5338
5339unsigned fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5340 if (RetVT.SimpleTy != MVT::v16i8)
5341 return 0;
5342 if ((Subtarget->hasMVEIntegerOps())) {
5343 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
5344 }
5345 if ((Subtarget->hasNEON())) {
5346 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
5347 }
5348 return 0;
5349}
5350
5351unsigned fastEmit_ISD_OR_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5352 if (RetVT.SimpleTy != MVT::v4i16)
5353 return 0;
5354 if ((Subtarget->hasNEON())) {
5355 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
5356 }
5357 return 0;
5358}
5359
5360unsigned fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5361 if (RetVT.SimpleTy != MVT::v8i16)
5362 return 0;
5363 if ((Subtarget->hasMVEIntegerOps())) {
5364 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
5365 }
5366 if ((Subtarget->hasNEON())) {
5367 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
5368 }
5369 return 0;
5370}
5371
5372unsigned fastEmit_ISD_OR_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5373 if (RetVT.SimpleTy != MVT::v2i32)
5374 return 0;
5375 if ((Subtarget->hasNEON())) {
5376 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
5377 }
5378 return 0;
5379}
5380
5381unsigned fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5382 if (RetVT.SimpleTy != MVT::v4i32)
5383 return 0;
5384 if ((Subtarget->hasMVEIntegerOps())) {
5385 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
5386 }
5387 if ((Subtarget->hasNEON())) {
5388 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
5389 }
5390 return 0;
5391}
5392
5393unsigned fastEmit_ISD_OR_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5394 if (RetVT.SimpleTy != MVT::v1i64)
5395 return 0;
5396 if ((Subtarget->hasNEON())) {
5397 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
5398 }
5399 return 0;
5400}
5401
5402unsigned fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5403 if (RetVT.SimpleTy != MVT::v2i64)
5404 return 0;
5405 if ((Subtarget->hasMVEIntegerOps())) {
5406 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
5407 }
5408 if ((Subtarget->hasNEON())) {
5409 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
5410 }
5411 return 0;
5412}
5413
5414unsigned fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
5415 switch (VT.SimpleTy) {
5416 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
5417 case MVT::v8i8: return fastEmit_ISD_OR_MVT_v8i8_rr(RetVT, Op0, Op1);
5418 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
5419 case MVT::v4i16: return fastEmit_ISD_OR_MVT_v4i16_rr(RetVT, Op0, Op1);
5420 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
5421 case MVT::v2i32: return fastEmit_ISD_OR_MVT_v2i32_rr(RetVT, Op0, Op1);
5422 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
5423 case MVT::v1i64: return fastEmit_ISD_OR_MVT_v1i64_rr(RetVT, Op0, Op1);
5424 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
5425 default: return 0;
5426 }
5427}
5428
5429// FastEmit functions for ISD::ROTR.
5430
5431unsigned fastEmit_ISD_ROTR_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5432 if (RetVT.SimpleTy != MVT::i32)
5433 return 0;
5434 if ((Subtarget->isThumb2())) {
5435 return fastEmitInst_rr(MachineInstOpcode: ARM::t2RORrr, RC: &ARM::rGPRRegClass, Op0, Op1);
5436 }
5437 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5438 return fastEmitInst_rr(MachineInstOpcode: ARM::tROR, RC: &ARM::tGPRRegClass, Op0, Op1);
5439 }
5440 return 0;
5441}
5442
5443unsigned fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
5444 switch (VT.SimpleTy) {
5445 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_rr(RetVT, Op0, Op1);
5446 default: return 0;
5447 }
5448}
5449
5450// FastEmit functions for ISD::SADDSAT.
5451
5452unsigned fastEmit_ISD_SADDSAT_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5453 if (RetVT.SimpleTy != MVT::i32)
5454 return 0;
5455 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
5456 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QADD, RC: &ARM::rGPRRegClass, Op0, Op1);
5457 }
5458 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
5459 return fastEmitInst_rr(MachineInstOpcode: ARM::QADD, RC: &ARM::GPRnopcRegClass, Op0, Op1);
5460 }
5461 return 0;
5462}
5463
5464unsigned fastEmit_ISD_SADDSAT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5465 if (RetVT.SimpleTy != MVT::v8i8)
5466 return 0;
5467 if ((Subtarget->hasNEON())) {
5468 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
5469 }
5470 return 0;
5471}
5472
5473unsigned fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5474 if (RetVT.SimpleTy != MVT::v16i8)
5475 return 0;
5476 if ((Subtarget->hasMVEIntegerOps())) {
5477 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
5478 }
5479 if ((Subtarget->hasNEON())) {
5480 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
5481 }
5482 return 0;
5483}
5484
5485unsigned fastEmit_ISD_SADDSAT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5486 if (RetVT.SimpleTy != MVT::v4i16)
5487 return 0;
5488 if ((Subtarget->hasNEON())) {
5489 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
5490 }
5491 return 0;
5492}
5493
5494unsigned fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5495 if (RetVT.SimpleTy != MVT::v8i16)
5496 return 0;
5497 if ((Subtarget->hasMVEIntegerOps())) {
5498 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
5499 }
5500 if ((Subtarget->hasNEON())) {
5501 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
5502 }
5503 return 0;
5504}
5505
5506unsigned fastEmit_ISD_SADDSAT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5507 if (RetVT.SimpleTy != MVT::v2i32)
5508 return 0;
5509 if ((Subtarget->hasNEON())) {
5510 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
5511 }
5512 return 0;
5513}
5514
5515unsigned fastEmit_ISD_SADDSAT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5516 if (RetVT.SimpleTy != MVT::v4i32)
5517 return 0;
5518 if ((Subtarget->hasMVEIntegerOps())) {
5519 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
5520 }
5521 if ((Subtarget->hasNEON())) {
5522 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
5523 }
5524 return 0;
5525}
5526
5527unsigned fastEmit_ISD_SADDSAT_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5528 if (RetVT.SimpleTy != MVT::v1i64)
5529 return 0;
5530 if ((Subtarget->hasNEON())) {
5531 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
5532 }
5533 return 0;
5534}
5535
5536unsigned fastEmit_ISD_SADDSAT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5537 if (RetVT.SimpleTy != MVT::v2i64)
5538 return 0;
5539 if ((Subtarget->hasNEON())) {
5540 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
5541 }
5542 return 0;
5543}
5544
5545unsigned fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
5546 switch (VT.SimpleTy) {
5547 case MVT::i32: return fastEmit_ISD_SADDSAT_MVT_i32_rr(RetVT, Op0, Op1);
5548 case MVT::v8i8: return fastEmit_ISD_SADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
5549 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
5550 case MVT::v4i16: return fastEmit_ISD_SADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
5551 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
5552 case MVT::v2i32: return fastEmit_ISD_SADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
5553 case MVT::v4i32: return fastEmit_ISD_SADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
5554 case MVT::v1i64: return fastEmit_ISD_SADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
5555 case MVT::v2i64: return fastEmit_ISD_SADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
5556 default: return 0;
5557 }
5558}
5559
5560// FastEmit functions for ISD::SDIV.
5561
5562unsigned fastEmit_ISD_SDIV_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5563 if (RetVT.SimpleTy != MVT::i32)
5564 return 0;
5565 if ((Subtarget->hasDivideInThumbMode()) && (Subtarget->hasV8MBaselineOps()) && (Subtarget->isThumb())) {
5566 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SDIV, RC: &ARM::rGPRRegClass, Op0, Op1);
5567 }
5568 if ((Subtarget->hasDivideInARMMode()) && (!Subtarget->isThumb())) {
5569 return fastEmitInst_rr(MachineInstOpcode: ARM::SDIV, RC: &ARM::GPRRegClass, Op0, Op1);
5570 }
5571 return 0;
5572}
5573
5574unsigned fastEmit_ISD_SDIV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
5575 switch (VT.SimpleTy) {
5576 case MVT::i32: return fastEmit_ISD_SDIV_MVT_i32_rr(RetVT, Op0, Op1);
5577 default: return 0;
5578 }
5579}
5580
5581// FastEmit functions for ISD::SHL.
5582
5583unsigned fastEmit_ISD_SHL_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5584 if (RetVT.SimpleTy != MVT::i32)
5585 return 0;
5586 if ((Subtarget->isThumb2())) {
5587 return fastEmitInst_rr(MachineInstOpcode: ARM::t2LSLrr, RC: &ARM::rGPRRegClass, Op0, Op1);
5588 }
5589 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5590 return fastEmitInst_rr(MachineInstOpcode: ARM::tLSLrr, RC: &ARM::tGPRRegClass, Op0, Op1);
5591 }
5592 return 0;
5593}
5594
5595unsigned fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
5596 switch (VT.SimpleTy) {
5597 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_rr(RetVT, Op0, Op1);
5598 default: return 0;
5599 }
5600}
5601
5602// FastEmit functions for ISD::SMAX.
5603
5604unsigned fastEmit_ISD_SMAX_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5605 if (RetVT.SimpleTy != MVT::v8i8)
5606 return 0;
5607 if ((Subtarget->hasNEON())) {
5608 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
5609 }
5610 return 0;
5611}
5612
5613unsigned fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5614 if (RetVT.SimpleTy != MVT::v16i8)
5615 return 0;
5616 if ((Subtarget->hasMVEIntegerOps())) {
5617 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXs8, RC: &ARM::MQPRRegClass, Op0, Op1);
5618 }
5619 if ((Subtarget->hasNEON())) {
5620 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
5621 }
5622 return 0;
5623}
5624
5625unsigned fastEmit_ISD_SMAX_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5626 if (RetVT.SimpleTy != MVT::v4i16)
5627 return 0;
5628 if ((Subtarget->hasNEON())) {
5629 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
5630 }
5631 return 0;
5632}
5633
5634unsigned fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5635 if (RetVT.SimpleTy != MVT::v8i16)
5636 return 0;
5637 if ((Subtarget->hasMVEIntegerOps())) {
5638 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXs16, RC: &ARM::MQPRRegClass, Op0, Op1);
5639 }
5640 if ((Subtarget->hasNEON())) {
5641 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
5642 }
5643 return 0;
5644}
5645
5646unsigned fastEmit_ISD_SMAX_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5647 if (RetVT.SimpleTy != MVT::v2i32)
5648 return 0;
5649 if ((Subtarget->hasNEON())) {
5650 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
5651 }
5652 return 0;
5653}
5654
5655unsigned fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5656 if (RetVT.SimpleTy != MVT::v4i32)
5657 return 0;
5658 if ((Subtarget->hasMVEIntegerOps())) {
5659 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXs32, RC: &ARM::MQPRRegClass, Op0, Op1);
5660 }
5661 if ((Subtarget->hasNEON())) {
5662 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
5663 }
5664 return 0;
5665}
5666
5667unsigned fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
5668 switch (VT.SimpleTy) {
5669 case MVT::v8i8: return fastEmit_ISD_SMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
5670 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
5671 case MVT::v4i16: return fastEmit_ISD_SMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
5672 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
5673 case MVT::v2i32: return fastEmit_ISD_SMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
5674 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
5675 default: return 0;
5676 }
5677}
5678
5679// FastEmit functions for ISD::SMIN.
5680
5681unsigned fastEmit_ISD_SMIN_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5682 if (RetVT.SimpleTy != MVT::v8i8)
5683 return 0;
5684 if ((Subtarget->hasNEON())) {
5685 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
5686 }
5687 return 0;
5688}
5689
5690unsigned fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5691 if (RetVT.SimpleTy != MVT::v16i8)
5692 return 0;
5693 if ((Subtarget->hasMVEIntegerOps())) {
5694 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINs8, RC: &ARM::MQPRRegClass, Op0, Op1);
5695 }
5696 if ((Subtarget->hasNEON())) {
5697 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
5698 }
5699 return 0;
5700}
5701
5702unsigned fastEmit_ISD_SMIN_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5703 if (RetVT.SimpleTy != MVT::v4i16)
5704 return 0;
5705 if ((Subtarget->hasNEON())) {
5706 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
5707 }
5708 return 0;
5709}
5710
5711unsigned fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5712 if (RetVT.SimpleTy != MVT::v8i16)
5713 return 0;
5714 if ((Subtarget->hasMVEIntegerOps())) {
5715 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINs16, RC: &ARM::MQPRRegClass, Op0, Op1);
5716 }
5717 if ((Subtarget->hasNEON())) {
5718 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
5719 }
5720 return 0;
5721}
5722
5723unsigned fastEmit_ISD_SMIN_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5724 if (RetVT.SimpleTy != MVT::v2i32)
5725 return 0;
5726 if ((Subtarget->hasNEON())) {
5727 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
5728 }
5729 return 0;
5730}
5731
5732unsigned fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5733 if (RetVT.SimpleTy != MVT::v4i32)
5734 return 0;
5735 if ((Subtarget->hasMVEIntegerOps())) {
5736 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINs32, RC: &ARM::MQPRRegClass, Op0, Op1);
5737 }
5738 if ((Subtarget->hasNEON())) {
5739 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
5740 }
5741 return 0;
5742}
5743
5744unsigned fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
5745 switch (VT.SimpleTy) {
5746 case MVT::v8i8: return fastEmit_ISD_SMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
5747 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
5748 case MVT::v4i16: return fastEmit_ISD_SMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
5749 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
5750 case MVT::v2i32: return fastEmit_ISD_SMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
5751 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
5752 default: return 0;
5753 }
5754}
5755
5756// FastEmit functions for ISD::SRA.
5757
5758unsigned fastEmit_ISD_SRA_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5759 if (RetVT.SimpleTy != MVT::i32)
5760 return 0;
5761 if ((Subtarget->isThumb2())) {
5762 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ASRrr, RC: &ARM::rGPRRegClass, Op0, Op1);
5763 }
5764 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5765 return fastEmitInst_rr(MachineInstOpcode: ARM::tASRrr, RC: &ARM::tGPRRegClass, Op0, Op1);
5766 }
5767 return 0;
5768}
5769
5770unsigned fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
5771 switch (VT.SimpleTy) {
5772 case MVT::i32: return fastEmit_ISD_SRA_MVT_i32_rr(RetVT, Op0, Op1);
5773 default: return 0;
5774 }
5775}
5776
5777// FastEmit functions for ISD::SRL.
5778
5779unsigned fastEmit_ISD_SRL_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5780 if (RetVT.SimpleTy != MVT::i32)
5781 return 0;
5782 if ((Subtarget->isThumb2())) {
5783 return fastEmitInst_rr(MachineInstOpcode: ARM::t2LSRrr, RC: &ARM::rGPRRegClass, Op0, Op1);
5784 }
5785 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5786 return fastEmitInst_rr(MachineInstOpcode: ARM::tLSRrr, RC: &ARM::tGPRRegClass, Op0, Op1);
5787 }
5788 return 0;
5789}
5790
5791unsigned fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
5792 switch (VT.SimpleTy) {
5793 case MVT::i32: return fastEmit_ISD_SRL_MVT_i32_rr(RetVT, Op0, Op1);
5794 default: return 0;
5795 }
5796}
5797
5798// FastEmit functions for ISD::SSUBSAT.
5799
5800unsigned fastEmit_ISD_SSUBSAT_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5801 if (RetVT.SimpleTy != MVT::i32)
5802 return 0;
5803 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
5804 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QSUB, RC: &ARM::rGPRRegClass, Op0, Op1);
5805 }
5806 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
5807 return fastEmitInst_rr(MachineInstOpcode: ARM::QSUB, RC: &ARM::GPRnopcRegClass, Op0, Op1);
5808 }
5809 return 0;
5810}
5811
5812unsigned fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5813 if (RetVT.SimpleTy != MVT::v8i8)
5814 return 0;
5815 if ((Subtarget->hasNEON())) {
5816 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
5817 }
5818 return 0;
5819}
5820
5821unsigned fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5822 if (RetVT.SimpleTy != MVT::v16i8)
5823 return 0;
5824 if ((Subtarget->hasMVEIntegerOps())) {
5825 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBs8, RC: &ARM::MQPRRegClass, Op0, Op1);
5826 }
5827 if ((Subtarget->hasNEON())) {
5828 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
5829 }
5830 return 0;
5831}
5832
5833unsigned fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5834 if (RetVT.SimpleTy != MVT::v4i16)
5835 return 0;
5836 if ((Subtarget->hasNEON())) {
5837 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
5838 }
5839 return 0;
5840}
5841
5842unsigned fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5843 if (RetVT.SimpleTy != MVT::v8i16)
5844 return 0;
5845 if ((Subtarget->hasMVEIntegerOps())) {
5846 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBs16, RC: &ARM::MQPRRegClass, Op0, Op1);
5847 }
5848 if ((Subtarget->hasNEON())) {
5849 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
5850 }
5851 return 0;
5852}
5853
5854unsigned fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5855 if (RetVT.SimpleTy != MVT::v2i32)
5856 return 0;
5857 if ((Subtarget->hasNEON())) {
5858 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
5859 }
5860 return 0;
5861}
5862
5863unsigned fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5864 if (RetVT.SimpleTy != MVT::v4i32)
5865 return 0;
5866 if ((Subtarget->hasMVEIntegerOps())) {
5867 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBs32, RC: &ARM::MQPRRegClass, Op0, Op1);
5868 }
5869 if ((Subtarget->hasNEON())) {
5870 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
5871 }
5872 return 0;
5873}
5874
5875unsigned fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5876 if (RetVT.SimpleTy != MVT::v1i64)
5877 return 0;
5878 if ((Subtarget->hasNEON())) {
5879 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
5880 }
5881 return 0;
5882}
5883
5884unsigned fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5885 if (RetVT.SimpleTy != MVT::v2i64)
5886 return 0;
5887 if ((Subtarget->hasNEON())) {
5888 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
5889 }
5890 return 0;
5891}
5892
5893unsigned fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
5894 switch (VT.SimpleTy) {
5895 case MVT::i32: return fastEmit_ISD_SSUBSAT_MVT_i32_rr(RetVT, Op0, Op1);
5896 case MVT::v8i8: return fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
5897 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
5898 case MVT::v4i16: return fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
5899 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
5900 case MVT::v2i32: return fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
5901 case MVT::v4i32: return fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
5902 case MVT::v1i64: return fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
5903 case MVT::v2i64: return fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
5904 default: return 0;
5905 }
5906}
5907
5908// FastEmit functions for ISD::SUB.
5909
5910unsigned fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5911 if (RetVT.SimpleTy != MVT::i32)
5912 return 0;
5913 if ((Subtarget->isThumb2())) {
5914 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SUBrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
5915 }
5916 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5917 return fastEmitInst_rr(MachineInstOpcode: ARM::tSUBrr, RC: &ARM::tGPRRegClass, Op0, Op1);
5918 }
5919 if ((!Subtarget->isThumb())) {
5920 return fastEmitInst_rr(MachineInstOpcode: ARM::SUBrr, RC: &ARM::GPRRegClass, Op0, Op1);
5921 }
5922 return 0;
5923}
5924
5925unsigned fastEmit_ISD_SUB_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5926 if (RetVT.SimpleTy != MVT::v8i8)
5927 return 0;
5928 if ((Subtarget->hasNEON())) {
5929 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
5930 }
5931 return 0;
5932}
5933
5934unsigned fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5935 if (RetVT.SimpleTy != MVT::v16i8)
5936 return 0;
5937 if ((Subtarget->hasMVEIntegerOps())) {
5938 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBi8, RC: &ARM::MQPRRegClass, Op0, Op1);
5939 }
5940 if ((Subtarget->hasNEON())) {
5941 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
5942 }
5943 return 0;
5944}
5945
5946unsigned fastEmit_ISD_SUB_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5947 if (RetVT.SimpleTy != MVT::v4i16)
5948 return 0;
5949 if ((Subtarget->hasNEON())) {
5950 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
5951 }
5952 return 0;
5953}
5954
5955unsigned fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5956 if (RetVT.SimpleTy != MVT::v8i16)
5957 return 0;
5958 if ((Subtarget->hasMVEIntegerOps())) {
5959 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBi16, RC: &ARM::MQPRRegClass, Op0, Op1);
5960 }
5961 if ((Subtarget->hasNEON())) {
5962 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
5963 }
5964 return 0;
5965}
5966
5967unsigned fastEmit_ISD_SUB_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5968 if (RetVT.SimpleTy != MVT::v2i32)
5969 return 0;
5970 if ((Subtarget->hasNEON())) {
5971 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
5972 }
5973 return 0;
5974}
5975
5976unsigned fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5977 if (RetVT.SimpleTy != MVT::v4i32)
5978 return 0;
5979 if ((Subtarget->hasMVEIntegerOps())) {
5980 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBi32, RC: &ARM::MQPRRegClass, Op0, Op1);
5981 }
5982 if ((Subtarget->hasNEON())) {
5983 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
5984 }
5985 return 0;
5986}
5987
5988unsigned fastEmit_ISD_SUB_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5989 if (RetVT.SimpleTy != MVT::v1i64)
5990 return 0;
5991 if ((Subtarget->hasNEON())) {
5992 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
5993 }
5994 return 0;
5995}
5996
5997unsigned fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
5998 if (RetVT.SimpleTy != MVT::v2i64)
5999 return 0;
6000 if ((Subtarget->hasNEON())) {
6001 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
6002 }
6003 return 0;
6004}
6005
6006unsigned fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
6007 switch (VT.SimpleTy) {
6008 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
6009 case MVT::v8i8: return fastEmit_ISD_SUB_MVT_v8i8_rr(RetVT, Op0, Op1);
6010 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
6011 case MVT::v4i16: return fastEmit_ISD_SUB_MVT_v4i16_rr(RetVT, Op0, Op1);
6012 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
6013 case MVT::v2i32: return fastEmit_ISD_SUB_MVT_v2i32_rr(RetVT, Op0, Op1);
6014 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
6015 case MVT::v1i64: return fastEmit_ISD_SUB_MVT_v1i64_rr(RetVT, Op0, Op1);
6016 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
6017 default: return 0;
6018 }
6019}
6020
6021// FastEmit functions for ISD::UADDSAT.
6022
6023unsigned fastEmit_ISD_UADDSAT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6024 if (RetVT.SimpleTy != MVT::v8i8)
6025 return 0;
6026 if ((Subtarget->hasNEON())) {
6027 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6028 }
6029 return 0;
6030}
6031
6032unsigned fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6033 if (RetVT.SimpleTy != MVT::v16i8)
6034 return 0;
6035 if ((Subtarget->hasMVEIntegerOps())) {
6036 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
6037 }
6038 if ((Subtarget->hasNEON())) {
6039 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6040 }
6041 return 0;
6042}
6043
6044unsigned fastEmit_ISD_UADDSAT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6045 if (RetVT.SimpleTy != MVT::v4i16)
6046 return 0;
6047 if ((Subtarget->hasNEON())) {
6048 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6049 }
6050 return 0;
6051}
6052
6053unsigned fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6054 if (RetVT.SimpleTy != MVT::v8i16)
6055 return 0;
6056 if ((Subtarget->hasMVEIntegerOps())) {
6057 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
6058 }
6059 if ((Subtarget->hasNEON())) {
6060 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6061 }
6062 return 0;
6063}
6064
6065unsigned fastEmit_ISD_UADDSAT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6066 if (RetVT.SimpleTy != MVT::v2i32)
6067 return 0;
6068 if ((Subtarget->hasNEON())) {
6069 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6070 }
6071 return 0;
6072}
6073
6074unsigned fastEmit_ISD_UADDSAT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6075 if (RetVT.SimpleTy != MVT::v4i32)
6076 return 0;
6077 if ((Subtarget->hasMVEIntegerOps())) {
6078 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
6079 }
6080 if ((Subtarget->hasNEON())) {
6081 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6082 }
6083 return 0;
6084}
6085
6086unsigned fastEmit_ISD_UADDSAT_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6087 if (RetVT.SimpleTy != MVT::v1i64)
6088 return 0;
6089 if ((Subtarget->hasNEON())) {
6090 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
6091 }
6092 return 0;
6093}
6094
6095unsigned fastEmit_ISD_UADDSAT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6096 if (RetVT.SimpleTy != MVT::v2i64)
6097 return 0;
6098 if ((Subtarget->hasNEON())) {
6099 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
6100 }
6101 return 0;
6102}
6103
6104unsigned fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
6105 switch (VT.SimpleTy) {
6106 case MVT::v8i8: return fastEmit_ISD_UADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
6107 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
6108 case MVT::v4i16: return fastEmit_ISD_UADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
6109 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
6110 case MVT::v2i32: return fastEmit_ISD_UADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
6111 case MVT::v4i32: return fastEmit_ISD_UADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
6112 case MVT::v1i64: return fastEmit_ISD_UADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
6113 case MVT::v2i64: return fastEmit_ISD_UADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
6114 default: return 0;
6115 }
6116}
6117
6118// FastEmit functions for ISD::UDIV.
6119
6120unsigned fastEmit_ISD_UDIV_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6121 if (RetVT.SimpleTy != MVT::i32)
6122 return 0;
6123 if ((Subtarget->hasDivideInThumbMode()) && (Subtarget->hasV8MBaselineOps()) && (Subtarget->isThumb())) {
6124 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UDIV, RC: &ARM::rGPRRegClass, Op0, Op1);
6125 }
6126 if ((Subtarget->hasDivideInARMMode()) && (!Subtarget->isThumb())) {
6127 return fastEmitInst_rr(MachineInstOpcode: ARM::UDIV, RC: &ARM::GPRRegClass, Op0, Op1);
6128 }
6129 return 0;
6130}
6131
6132unsigned fastEmit_ISD_UDIV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
6133 switch (VT.SimpleTy) {
6134 case MVT::i32: return fastEmit_ISD_UDIV_MVT_i32_rr(RetVT, Op0, Op1);
6135 default: return 0;
6136 }
6137}
6138
6139// FastEmit functions for ISD::UMAX.
6140
6141unsigned fastEmit_ISD_UMAX_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6142 if (RetVT.SimpleTy != MVT::v8i8)
6143 return 0;
6144 if ((Subtarget->hasNEON())) {
6145 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6146 }
6147 return 0;
6148}
6149
6150unsigned fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6151 if (RetVT.SimpleTy != MVT::v16i8)
6152 return 0;
6153 if ((Subtarget->hasMVEIntegerOps())) {
6154 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXu8, RC: &ARM::MQPRRegClass, Op0, Op1);
6155 }
6156 if ((Subtarget->hasNEON())) {
6157 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6158 }
6159 return 0;
6160}
6161
6162unsigned fastEmit_ISD_UMAX_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6163 if (RetVT.SimpleTy != MVT::v4i16)
6164 return 0;
6165 if ((Subtarget->hasNEON())) {
6166 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6167 }
6168 return 0;
6169}
6170
6171unsigned fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6172 if (RetVT.SimpleTy != MVT::v8i16)
6173 return 0;
6174 if ((Subtarget->hasMVEIntegerOps())) {
6175 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXu16, RC: &ARM::MQPRRegClass, Op0, Op1);
6176 }
6177 if ((Subtarget->hasNEON())) {
6178 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6179 }
6180 return 0;
6181}
6182
6183unsigned fastEmit_ISD_UMAX_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6184 if (RetVT.SimpleTy != MVT::v2i32)
6185 return 0;
6186 if ((Subtarget->hasNEON())) {
6187 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6188 }
6189 return 0;
6190}
6191
6192unsigned fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6193 if (RetVT.SimpleTy != MVT::v4i32)
6194 return 0;
6195 if ((Subtarget->hasMVEIntegerOps())) {
6196 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXu32, RC: &ARM::MQPRRegClass, Op0, Op1);
6197 }
6198 if ((Subtarget->hasNEON())) {
6199 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6200 }
6201 return 0;
6202}
6203
6204unsigned fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
6205 switch (VT.SimpleTy) {
6206 case MVT::v8i8: return fastEmit_ISD_UMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
6207 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
6208 case MVT::v4i16: return fastEmit_ISD_UMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
6209 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
6210 case MVT::v2i32: return fastEmit_ISD_UMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
6211 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
6212 default: return 0;
6213 }
6214}
6215
6216// FastEmit functions for ISD::UMIN.
6217
6218unsigned fastEmit_ISD_UMIN_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6219 if (RetVT.SimpleTy != MVT::v8i8)
6220 return 0;
6221 if ((Subtarget->hasNEON())) {
6222 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6223 }
6224 return 0;
6225}
6226
6227unsigned fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6228 if (RetVT.SimpleTy != MVT::v16i8)
6229 return 0;
6230 if ((Subtarget->hasMVEIntegerOps())) {
6231 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINu8, RC: &ARM::MQPRRegClass, Op0, Op1);
6232 }
6233 if ((Subtarget->hasNEON())) {
6234 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6235 }
6236 return 0;
6237}
6238
6239unsigned fastEmit_ISD_UMIN_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6240 if (RetVT.SimpleTy != MVT::v4i16)
6241 return 0;
6242 if ((Subtarget->hasNEON())) {
6243 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6244 }
6245 return 0;
6246}
6247
6248unsigned fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6249 if (RetVT.SimpleTy != MVT::v8i16)
6250 return 0;
6251 if ((Subtarget->hasMVEIntegerOps())) {
6252 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINu16, RC: &ARM::MQPRRegClass, Op0, Op1);
6253 }
6254 if ((Subtarget->hasNEON())) {
6255 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6256 }
6257 return 0;
6258}
6259
6260unsigned fastEmit_ISD_UMIN_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6261 if (RetVT.SimpleTy != MVT::v2i32)
6262 return 0;
6263 if ((Subtarget->hasNEON())) {
6264 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6265 }
6266 return 0;
6267}
6268
6269unsigned fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6270 if (RetVT.SimpleTy != MVT::v4i32)
6271 return 0;
6272 if ((Subtarget->hasMVEIntegerOps())) {
6273 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINu32, RC: &ARM::MQPRRegClass, Op0, Op1);
6274 }
6275 if ((Subtarget->hasNEON())) {
6276 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6277 }
6278 return 0;
6279}
6280
6281unsigned fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
6282 switch (VT.SimpleTy) {
6283 case MVT::v8i8: return fastEmit_ISD_UMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
6284 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
6285 case MVT::v4i16: return fastEmit_ISD_UMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
6286 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
6287 case MVT::v2i32: return fastEmit_ISD_UMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
6288 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
6289 default: return 0;
6290 }
6291}
6292
6293// FastEmit functions for ISD::USUBSAT.
6294
6295unsigned fastEmit_ISD_USUBSAT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6296 if (RetVT.SimpleTy != MVT::v8i8)
6297 return 0;
6298 if ((Subtarget->hasNEON())) {
6299 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6300 }
6301 return 0;
6302}
6303
6304unsigned fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6305 if (RetVT.SimpleTy != MVT::v16i8)
6306 return 0;
6307 if ((Subtarget->hasMVEIntegerOps())) {
6308 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBu8, RC: &ARM::MQPRRegClass, Op0, Op1);
6309 }
6310 if ((Subtarget->hasNEON())) {
6311 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6312 }
6313 return 0;
6314}
6315
6316unsigned fastEmit_ISD_USUBSAT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6317 if (RetVT.SimpleTy != MVT::v4i16)
6318 return 0;
6319 if ((Subtarget->hasNEON())) {
6320 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6321 }
6322 return 0;
6323}
6324
6325unsigned fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6326 if (RetVT.SimpleTy != MVT::v8i16)
6327 return 0;
6328 if ((Subtarget->hasMVEIntegerOps())) {
6329 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBu16, RC: &ARM::MQPRRegClass, Op0, Op1);
6330 }
6331 if ((Subtarget->hasNEON())) {
6332 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6333 }
6334 return 0;
6335}
6336
6337unsigned fastEmit_ISD_USUBSAT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6338 if (RetVT.SimpleTy != MVT::v2i32)
6339 return 0;
6340 if ((Subtarget->hasNEON())) {
6341 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6342 }
6343 return 0;
6344}
6345
6346unsigned fastEmit_ISD_USUBSAT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6347 if (RetVT.SimpleTy != MVT::v4i32)
6348 return 0;
6349 if ((Subtarget->hasMVEIntegerOps())) {
6350 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBu32, RC: &ARM::MQPRRegClass, Op0, Op1);
6351 }
6352 if ((Subtarget->hasNEON())) {
6353 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6354 }
6355 return 0;
6356}
6357
6358unsigned fastEmit_ISD_USUBSAT_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6359 if (RetVT.SimpleTy != MVT::v1i64)
6360 return 0;
6361 if ((Subtarget->hasNEON())) {
6362 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
6363 }
6364 return 0;
6365}
6366
6367unsigned fastEmit_ISD_USUBSAT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6368 if (RetVT.SimpleTy != MVT::v2i64)
6369 return 0;
6370 if ((Subtarget->hasNEON())) {
6371 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
6372 }
6373 return 0;
6374}
6375
6376unsigned fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
6377 switch (VT.SimpleTy) {
6378 case MVT::v8i8: return fastEmit_ISD_USUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
6379 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
6380 case MVT::v4i16: return fastEmit_ISD_USUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
6381 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
6382 case MVT::v2i32: return fastEmit_ISD_USUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
6383 case MVT::v4i32: return fastEmit_ISD_USUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
6384 case MVT::v1i64: return fastEmit_ISD_USUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
6385 case MVT::v2i64: return fastEmit_ISD_USUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
6386 default: return 0;
6387 }
6388}
6389
6390// FastEmit functions for ISD::XOR.
6391
6392unsigned fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6393 if (RetVT.SimpleTy != MVT::i32)
6394 return 0;
6395 if ((Subtarget->isThumb2())) {
6396 return fastEmitInst_rr(MachineInstOpcode: ARM::t2EORrr, RC: &ARM::rGPRRegClass, Op0, Op1);
6397 }
6398 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6399 return fastEmitInst_rr(MachineInstOpcode: ARM::tEOR, RC: &ARM::tGPRRegClass, Op0, Op1);
6400 }
6401 if ((!Subtarget->isThumb())) {
6402 return fastEmitInst_rr(MachineInstOpcode: ARM::EORrr, RC: &ARM::GPRRegClass, Op0, Op1);
6403 }
6404 return 0;
6405}
6406
6407unsigned fastEmit_ISD_XOR_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6408 if (RetVT.SimpleTy != MVT::v8i8)
6409 return 0;
6410 if ((Subtarget->hasNEON())) {
6411 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
6412 }
6413 return 0;
6414}
6415
6416unsigned fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6417 if (RetVT.SimpleTy != MVT::v16i8)
6418 return 0;
6419 if ((Subtarget->hasMVEIntegerOps())) {
6420 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
6421 }
6422 if ((Subtarget->hasNEON())) {
6423 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
6424 }
6425 return 0;
6426}
6427
6428unsigned fastEmit_ISD_XOR_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6429 if (RetVT.SimpleTy != MVT::v4i16)
6430 return 0;
6431 if ((Subtarget->hasNEON())) {
6432 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
6433 }
6434 return 0;
6435}
6436
6437unsigned fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6438 if (RetVT.SimpleTy != MVT::v8i16)
6439 return 0;
6440 if ((Subtarget->hasMVEIntegerOps())) {
6441 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
6442 }
6443 if ((Subtarget->hasNEON())) {
6444 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
6445 }
6446 return 0;
6447}
6448
6449unsigned fastEmit_ISD_XOR_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6450 if (RetVT.SimpleTy != MVT::v2i32)
6451 return 0;
6452 if ((Subtarget->hasNEON())) {
6453 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
6454 }
6455 return 0;
6456}
6457
6458unsigned fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6459 if (RetVT.SimpleTy != MVT::v4i32)
6460 return 0;
6461 if ((Subtarget->hasMVEIntegerOps())) {
6462 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
6463 }
6464 if ((Subtarget->hasNEON())) {
6465 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
6466 }
6467 return 0;
6468}
6469
6470unsigned fastEmit_ISD_XOR_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6471 if (RetVT.SimpleTy != MVT::v1i64)
6472 return 0;
6473 if ((Subtarget->hasNEON())) {
6474 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
6475 }
6476 return 0;
6477}
6478
6479unsigned fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
6480 if (RetVT.SimpleTy != MVT::v2i64)
6481 return 0;
6482 if ((Subtarget->hasMVEIntegerOps())) {
6483 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
6484 }
6485 if ((Subtarget->hasNEON())) {
6486 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
6487 }
6488 return 0;
6489}
6490
6491unsigned fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
6492 switch (VT.SimpleTy) {
6493 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
6494 case MVT::v8i8: return fastEmit_ISD_XOR_MVT_v8i8_rr(RetVT, Op0, Op1);
6495 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
6496 case MVT::v4i16: return fastEmit_ISD_XOR_MVT_v4i16_rr(RetVT, Op0, Op1);
6497 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
6498 case MVT::v2i32: return fastEmit_ISD_XOR_MVT_v2i32_rr(RetVT, Op0, Op1);
6499 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
6500 case MVT::v1i64: return fastEmit_ISD_XOR_MVT_v1i64_rr(RetVT, Op0, Op1);
6501 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
6502 default: return 0;
6503 }
6504}
6505
6506// Top-level FastEmit function.
6507
6508unsigned fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, unsigned Op1) override {
6509 switch (Opcode) {
6510 case ARMISD::CMP: return fastEmit_ARMISD_CMP_rr(VT, RetVT, Op0, Op1);
6511 case ARMISD::CMPFP: return fastEmit_ARMISD_CMPFP_rr(VT, RetVT, Op0, Op1);
6512 case ARMISD::CMPFPE: return fastEmit_ARMISD_CMPFPE_rr(VT, RetVT, Op0, Op1);
6513 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_rr(VT, RetVT, Op0, Op1);
6514 case ARMISD::EH_SJLJ_LONGJMP: return fastEmit_ARMISD_EH_SJLJ_LONGJMP_rr(VT, RetVT, Op0, Op1);
6515 case ARMISD::EH_SJLJ_SETJMP: return fastEmit_ARMISD_EH_SJLJ_SETJMP_rr(VT, RetVT, Op0, Op1);
6516 case ARMISD::QADD16b: return fastEmit_ARMISD_QADD16b_rr(VT, RetVT, Op0, Op1);
6517 case ARMISD::QADD8b: return fastEmit_ARMISD_QADD8b_rr(VT, RetVT, Op0, Op1);
6518 case ARMISD::QSUB16b: return fastEmit_ARMISD_QSUB16b_rr(VT, RetVT, Op0, Op1);
6519 case ARMISD::QSUB8b: return fastEmit_ARMISD_QSUB8b_rr(VT, RetVT, Op0, Op1);
6520 case ARMISD::SMULWB: return fastEmit_ARMISD_SMULWB_rr(VT, RetVT, Op0, Op1);
6521 case ARMISD::SMULWT: return fastEmit_ARMISD_SMULWT_rr(VT, RetVT, Op0, Op1);
6522 case ARMISD::UQADD16b: return fastEmit_ARMISD_UQADD16b_rr(VT, RetVT, Op0, Op1);
6523 case ARMISD::UQADD8b: return fastEmit_ARMISD_UQADD8b_rr(VT, RetVT, Op0, Op1);
6524 case ARMISD::UQSUB16b: return fastEmit_ARMISD_UQSUB16b_rr(VT, RetVT, Op0, Op1);
6525 case ARMISD::UQSUB8b: return fastEmit_ARMISD_UQSUB8b_rr(VT, RetVT, Op0, Op1);
6526 case ARMISD::VMLAVs: return fastEmit_ARMISD_VMLAVs_rr(VT, RetVT, Op0, Op1);
6527 case ARMISD::VMLAVu: return fastEmit_ARMISD_VMLAVu_rr(VT, RetVT, Op0, Op1);
6528 case ARMISD::VMOVDRR: return fastEmit_ARMISD_VMOVDRR_rr(VT, RetVT, Op0, Op1);
6529 case ARMISD::VMULLs: return fastEmit_ARMISD_VMULLs_rr(VT, RetVT, Op0, Op1);
6530 case ARMISD::VMULLu: return fastEmit_ARMISD_VMULLu_rr(VT, RetVT, Op0, Op1);
6531 case ARMISD::VQDMULH: return fastEmit_ARMISD_VQDMULH_rr(VT, RetVT, Op0, Op1);
6532 case ARMISD::VSHLs: return fastEmit_ARMISD_VSHLs_rr(VT, RetVT, Op0, Op1);
6533 case ARMISD::VSHLu: return fastEmit_ARMISD_VSHLu_rr(VT, RetVT, Op0, Op1);
6534 case ARMISD::VTBL1: return fastEmit_ARMISD_VTBL1_rr(VT, RetVT, Op0, Op1);
6535 case ARMISD::VTST: return fastEmit_ARMISD_VTST_rr(VT, RetVT, Op0, Op1);
6536 case ISD::ABDS: return fastEmit_ISD_ABDS_rr(VT, RetVT, Op0, Op1);
6537 case ISD::ABDU: return fastEmit_ISD_ABDU_rr(VT, RetVT, Op0, Op1);
6538 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
6539 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
6540 case ISD::AVGCEILS: return fastEmit_ISD_AVGCEILS_rr(VT, RetVT, Op0, Op1);
6541 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
6542 case ISD::AVGFLOORS: return fastEmit_ISD_AVGFLOORS_rr(VT, RetVT, Op0, Op1);
6543 case ISD::AVGFLOORU: return fastEmit_ISD_AVGFLOORU_rr(VT, RetVT, Op0, Op1);
6544 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
6545 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
6546 case ISD::FMAXIMUM: return fastEmit_ISD_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
6547 case ISD::FMAXNUM: return fastEmit_ISD_FMAXNUM_rr(VT, RetVT, Op0, Op1);
6548 case ISD::FMINIMUM: return fastEmit_ISD_FMINIMUM_rr(VT, RetVT, Op0, Op1);
6549 case ISD::FMINNUM: return fastEmit_ISD_FMINNUM_rr(VT, RetVT, Op0, Op1);
6550 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
6551 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
6552 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
6553 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
6554 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
6555 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
6556 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
6557 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
6558 case ISD::SDIV: return fastEmit_ISD_SDIV_rr(VT, RetVT, Op0, Op1);
6559 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
6560 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
6561 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
6562 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
6563 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
6564 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
6565 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
6566 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
6567 case ISD::UDIV: return fastEmit_ISD_UDIV_rr(VT, RetVT, Op0, Op1);
6568 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
6569 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
6570 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
6571 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
6572 default: return 0;
6573 }
6574}
6575
6576// FastEmit functions for ARMISD::PIC_ADD.
6577
6578unsigned fastEmit_ARMISD_PIC_ADD_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6579 if (RetVT.SimpleTy != MVT::i32)
6580 return 0;
6581 if ((Subtarget->isThumb())) {
6582 return fastEmitInst_ri(MachineInstOpcode: ARM::tPICADD, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
6583 }
6584 if ((!Subtarget->isThumb())) {
6585 return fastEmitInst_ri(MachineInstOpcode: ARM::PICADD, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
6586 }
6587 return 0;
6588}
6589
6590unsigned fastEmit_ARMISD_PIC_ADD_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
6591 switch (VT.SimpleTy) {
6592 case MVT::i32: return fastEmit_ARMISD_PIC_ADD_MVT_i32_ri(RetVT, Op0, imm1);
6593 default: return 0;
6594 }
6595}
6596
6597// FastEmit functions for ARMISD::VDUPLANE.
6598
6599unsigned fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6600 if (RetVT.SimpleTy != MVT::v8i8)
6601 return 0;
6602 if ((Subtarget->hasNEON())) {
6603 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN8d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6604 }
6605 return 0;
6606}
6607
6608unsigned fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6609 if (RetVT.SimpleTy != MVT::v4i16)
6610 return 0;
6611 if ((Subtarget->hasNEON())) {
6612 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6613 }
6614 return 0;
6615}
6616
6617unsigned fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6618 if (RetVT.SimpleTy != MVT::v2i32)
6619 return 0;
6620 if ((Subtarget->hasNEON())) {
6621 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6622 }
6623 return 0;
6624}
6625
6626unsigned fastEmit_ARMISD_VDUPLANE_MVT_v4f16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6627 if (RetVT.SimpleTy != MVT::v4f16)
6628 return 0;
6629 if ((Subtarget->hasNEON())) {
6630 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6631 }
6632 return 0;
6633}
6634
6635unsigned fastEmit_ARMISD_VDUPLANE_MVT_v4bf16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6636 if (RetVT.SimpleTy != MVT::v4bf16)
6637 return 0;
6638 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
6639 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6640 }
6641 return 0;
6642}
6643
6644unsigned fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v2f32_ri(unsigned Op0, uint64_t imm1) {
6645 if ((Subtarget->hasNEON())) {
6646 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6647 }
6648 return 0;
6649}
6650
6651unsigned fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v4f32_ri(unsigned Op0, uint64_t imm1) {
6652 if ((Subtarget->hasNEON())) {
6653 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
6654 }
6655 return 0;
6656}
6657
6658unsigned fastEmit_ARMISD_VDUPLANE_MVT_v2f32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6659switch (RetVT.SimpleTy) {
6660 case MVT::v2f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v2f32_ri(Op0, imm1);
6661 case MVT::v4f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v4f32_ri(Op0, imm1);
6662 default: return 0;
6663}
6664}
6665
6666unsigned fastEmit_ARMISD_VDUPLANE_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
6667 switch (VT.SimpleTy) {
6668 case MVT::v8i8: return fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri(RetVT, Op0, imm1);
6669 case MVT::v4i16: return fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri(RetVT, Op0, imm1);
6670 case MVT::v2i32: return fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri(RetVT, Op0, imm1);
6671 case MVT::v4f16: return fastEmit_ARMISD_VDUPLANE_MVT_v4f16_ri(RetVT, Op0, imm1);
6672 case MVT::v4bf16: return fastEmit_ARMISD_VDUPLANE_MVT_v4bf16_ri(RetVT, Op0, imm1);
6673 case MVT::v2f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_ri(RetVT, Op0, imm1);
6674 default: return 0;
6675 }
6676}
6677
6678// FastEmit functions for ARMISD::VGETLANEs.
6679
6680unsigned fastEmit_ARMISD_VGETLANEs_MVT_v8i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6681 if (RetVT.SimpleTy != MVT::i32)
6682 return 0;
6683 if ((Subtarget->hasNEON())) {
6684 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNs8, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
6685 }
6686 return 0;
6687}
6688
6689unsigned fastEmit_ARMISD_VGETLANEs_MVT_v16i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6690 if (RetVT.SimpleTy != MVT::i32)
6691 return 0;
6692 if ((Subtarget->hasMVEIntegerOps())) {
6693 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_s8, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
6694 }
6695 return 0;
6696}
6697
6698unsigned fastEmit_ARMISD_VGETLANEs_MVT_v4i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6699 if (RetVT.SimpleTy != MVT::i32)
6700 return 0;
6701 if ((Subtarget->hasNEON())) {
6702 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNs16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
6703 }
6704 return 0;
6705}
6706
6707unsigned fastEmit_ARMISD_VGETLANEs_MVT_v8i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6708 if (RetVT.SimpleTy != MVT::i32)
6709 return 0;
6710 if ((Subtarget->hasMVEIntegerOps())) {
6711 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_s16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
6712 }
6713 return 0;
6714}
6715
6716unsigned fastEmit_ARMISD_VGETLANEs_MVT_v8f16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6717 if (RetVT.SimpleTy != MVT::i32)
6718 return 0;
6719 if ((Subtarget->hasMVEIntegerOps())) {
6720 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_s16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
6721 }
6722 return 0;
6723}
6724
6725unsigned fastEmit_ARMISD_VGETLANEs_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
6726 switch (VT.SimpleTy) {
6727 case MVT::v8i8: return fastEmit_ARMISD_VGETLANEs_MVT_v8i8_ri(RetVT, Op0, imm1);
6728 case MVT::v16i8: return fastEmit_ARMISD_VGETLANEs_MVT_v16i8_ri(RetVT, Op0, imm1);
6729 case MVT::v4i16: return fastEmit_ARMISD_VGETLANEs_MVT_v4i16_ri(RetVT, Op0, imm1);
6730 case MVT::v8i16: return fastEmit_ARMISD_VGETLANEs_MVT_v8i16_ri(RetVT, Op0, imm1);
6731 case MVT::v8f16: return fastEmit_ARMISD_VGETLANEs_MVT_v8f16_ri(RetVT, Op0, imm1);
6732 default: return 0;
6733 }
6734}
6735
6736// FastEmit functions for ARMISD::VGETLANEu.
6737
6738unsigned fastEmit_ARMISD_VGETLANEu_MVT_v8i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6739 if (RetVT.SimpleTy != MVT::i32)
6740 return 0;
6741 if ((Subtarget->hasNEON())) {
6742 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu8, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
6743 }
6744 return 0;
6745}
6746
6747unsigned fastEmit_ARMISD_VGETLANEu_MVT_v16i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6748 if (RetVT.SimpleTy != MVT::i32)
6749 return 0;
6750 if ((Subtarget->hasMVEIntegerOps())) {
6751 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_u8, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
6752 }
6753 return 0;
6754}
6755
6756unsigned fastEmit_ARMISD_VGETLANEu_MVT_v4i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6757 if (RetVT.SimpleTy != MVT::i32)
6758 return 0;
6759 if ((Subtarget->hasNEON())) {
6760 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
6761 }
6762 return 0;
6763}
6764
6765unsigned fastEmit_ARMISD_VGETLANEu_MVT_v8i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6766 if (RetVT.SimpleTy != MVT::i32)
6767 return 0;
6768 if ((Subtarget->hasMVEIntegerOps())) {
6769 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_u16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
6770 }
6771 return 0;
6772}
6773
6774unsigned fastEmit_ARMISD_VGETLANEu_MVT_v4f16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6775 if (RetVT.SimpleTy != MVT::i32)
6776 return 0;
6777 if ((Subtarget->hasNEON())) {
6778 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
6779 }
6780 return 0;
6781}
6782
6783unsigned fastEmit_ARMISD_VGETLANEu_MVT_v8f16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6784 if (RetVT.SimpleTy != MVT::i32)
6785 return 0;
6786 if ((Subtarget->hasMVEIntegerOps())) {
6787 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_u16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
6788 }
6789 return 0;
6790}
6791
6792unsigned fastEmit_ARMISD_VGETLANEu_MVT_v4bf16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6793 if (RetVT.SimpleTy != MVT::i32)
6794 return 0;
6795 if ((Subtarget->hasNEON())) {
6796 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
6797 }
6798 return 0;
6799}
6800
6801unsigned fastEmit_ARMISD_VGETLANEu_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
6802 switch (VT.SimpleTy) {
6803 case MVT::v8i8: return fastEmit_ARMISD_VGETLANEu_MVT_v8i8_ri(RetVT, Op0, imm1);
6804 case MVT::v16i8: return fastEmit_ARMISD_VGETLANEu_MVT_v16i8_ri(RetVT, Op0, imm1);
6805 case MVT::v4i16: return fastEmit_ARMISD_VGETLANEu_MVT_v4i16_ri(RetVT, Op0, imm1);
6806 case MVT::v8i16: return fastEmit_ARMISD_VGETLANEu_MVT_v8i16_ri(RetVT, Op0, imm1);
6807 case MVT::v4f16: return fastEmit_ARMISD_VGETLANEu_MVT_v4f16_ri(RetVT, Op0, imm1);
6808 case MVT::v8f16: return fastEmit_ARMISD_VGETLANEu_MVT_v8f16_ri(RetVT, Op0, imm1);
6809 case MVT::v4bf16: return fastEmit_ARMISD_VGETLANEu_MVT_v4bf16_ri(RetVT, Op0, imm1);
6810 default: return 0;
6811 }
6812}
6813
6814// FastEmit functions for ARMISD::VQSHLsIMM.
6815
6816unsigned fastEmit_ARMISD_VQSHLsIMM_MVT_v8i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6817 if (RetVT.SimpleTy != MVT::v8i8)
6818 return 0;
6819 if ((Subtarget->hasNEON())) {
6820 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6821 }
6822 return 0;
6823}
6824
6825unsigned fastEmit_ARMISD_VQSHLsIMM_MVT_v16i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6826 if (RetVT.SimpleTy != MVT::v16i8)
6827 return 0;
6828 if ((Subtarget->hasNEON())) {
6829 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
6830 }
6831 return 0;
6832}
6833
6834unsigned fastEmit_ARMISD_VQSHLsIMM_MVT_v4i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6835 if (RetVT.SimpleTy != MVT::v4i16)
6836 return 0;
6837 if ((Subtarget->hasNEON())) {
6838 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6839 }
6840 return 0;
6841}
6842
6843unsigned fastEmit_ARMISD_VQSHLsIMM_MVT_v8i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6844 if (RetVT.SimpleTy != MVT::v8i16)
6845 return 0;
6846 if ((Subtarget->hasNEON())) {
6847 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
6848 }
6849 return 0;
6850}
6851
6852unsigned fastEmit_ARMISD_VQSHLsIMM_MVT_v2i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6853 if (RetVT.SimpleTy != MVT::v2i32)
6854 return 0;
6855 if ((Subtarget->hasNEON())) {
6856 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6857 }
6858 return 0;
6859}
6860
6861unsigned fastEmit_ARMISD_VQSHLsIMM_MVT_v4i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6862 if (RetVT.SimpleTy != MVT::v4i32)
6863 return 0;
6864 if ((Subtarget->hasNEON())) {
6865 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
6866 }
6867 return 0;
6868}
6869
6870unsigned fastEmit_ARMISD_VQSHLsIMM_MVT_v1i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6871 if (RetVT.SimpleTy != MVT::v1i64)
6872 return 0;
6873 if ((Subtarget->hasNEON())) {
6874 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6875 }
6876 return 0;
6877}
6878
6879unsigned fastEmit_ARMISD_VQSHLsIMM_MVT_v2i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6880 if (RetVT.SimpleTy != MVT::v2i64)
6881 return 0;
6882 if ((Subtarget->hasNEON())) {
6883 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
6884 }
6885 return 0;
6886}
6887
6888unsigned fastEmit_ARMISD_VQSHLsIMM_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
6889 switch (VT.SimpleTy) {
6890 case MVT::v8i8: return fastEmit_ARMISD_VQSHLsIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
6891 case MVT::v16i8: return fastEmit_ARMISD_VQSHLsIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
6892 case MVT::v4i16: return fastEmit_ARMISD_VQSHLsIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
6893 case MVT::v8i16: return fastEmit_ARMISD_VQSHLsIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
6894 case MVT::v2i32: return fastEmit_ARMISD_VQSHLsIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
6895 case MVT::v4i32: return fastEmit_ARMISD_VQSHLsIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
6896 case MVT::v1i64: return fastEmit_ARMISD_VQSHLsIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
6897 case MVT::v2i64: return fastEmit_ARMISD_VQSHLsIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
6898 default: return 0;
6899 }
6900}
6901
6902// FastEmit functions for ARMISD::VQSHLsuIMM.
6903
6904unsigned fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6905 if (RetVT.SimpleTy != MVT::v8i8)
6906 return 0;
6907 if ((Subtarget->hasNEON())) {
6908 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6909 }
6910 return 0;
6911}
6912
6913unsigned fastEmit_ARMISD_VQSHLsuIMM_MVT_v16i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6914 if (RetVT.SimpleTy != MVT::v16i8)
6915 return 0;
6916 if ((Subtarget->hasNEON())) {
6917 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
6918 }
6919 return 0;
6920}
6921
6922unsigned fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6923 if (RetVT.SimpleTy != MVT::v4i16)
6924 return 0;
6925 if ((Subtarget->hasNEON())) {
6926 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6927 }
6928 return 0;
6929}
6930
6931unsigned fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6932 if (RetVT.SimpleTy != MVT::v8i16)
6933 return 0;
6934 if ((Subtarget->hasNEON())) {
6935 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
6936 }
6937 return 0;
6938}
6939
6940unsigned fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6941 if (RetVT.SimpleTy != MVT::v2i32)
6942 return 0;
6943 if ((Subtarget->hasNEON())) {
6944 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6945 }
6946 return 0;
6947}
6948
6949unsigned fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6950 if (RetVT.SimpleTy != MVT::v4i32)
6951 return 0;
6952 if ((Subtarget->hasNEON())) {
6953 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
6954 }
6955 return 0;
6956}
6957
6958unsigned fastEmit_ARMISD_VQSHLsuIMM_MVT_v1i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6959 if (RetVT.SimpleTy != MVT::v1i64)
6960 return 0;
6961 if ((Subtarget->hasNEON())) {
6962 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6963 }
6964 return 0;
6965}
6966
6967unsigned fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6968 if (RetVT.SimpleTy != MVT::v2i64)
6969 return 0;
6970 if ((Subtarget->hasNEON())) {
6971 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
6972 }
6973 return 0;
6974}
6975
6976unsigned fastEmit_ARMISD_VQSHLsuIMM_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
6977 switch (VT.SimpleTy) {
6978 case MVT::v8i8: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
6979 case MVT::v16i8: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
6980 case MVT::v4i16: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
6981 case MVT::v8i16: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
6982 case MVT::v2i32: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
6983 case MVT::v4i32: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
6984 case MVT::v1i64: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
6985 case MVT::v2i64: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
6986 default: return 0;
6987 }
6988}
6989
6990// FastEmit functions for ARMISD::VQSHLuIMM.
6991
6992unsigned fastEmit_ARMISD_VQSHLuIMM_MVT_v8i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
6993 if (RetVT.SimpleTy != MVT::v8i8)
6994 return 0;
6995 if ((Subtarget->hasNEON())) {
6996 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
6997 }
6998 return 0;
6999}
7000
7001unsigned fastEmit_ARMISD_VQSHLuIMM_MVT_v16i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7002 if (RetVT.SimpleTy != MVT::v16i8)
7003 return 0;
7004 if ((Subtarget->hasNEON())) {
7005 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7006 }
7007 return 0;
7008}
7009
7010unsigned fastEmit_ARMISD_VQSHLuIMM_MVT_v4i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7011 if (RetVT.SimpleTy != MVT::v4i16)
7012 return 0;
7013 if ((Subtarget->hasNEON())) {
7014 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7015 }
7016 return 0;
7017}
7018
7019unsigned fastEmit_ARMISD_VQSHLuIMM_MVT_v8i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7020 if (RetVT.SimpleTy != MVT::v8i16)
7021 return 0;
7022 if ((Subtarget->hasNEON())) {
7023 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7024 }
7025 return 0;
7026}
7027
7028unsigned fastEmit_ARMISD_VQSHLuIMM_MVT_v2i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7029 if (RetVT.SimpleTy != MVT::v2i32)
7030 return 0;
7031 if ((Subtarget->hasNEON())) {
7032 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7033 }
7034 return 0;
7035}
7036
7037unsigned fastEmit_ARMISD_VQSHLuIMM_MVT_v4i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7038 if (RetVT.SimpleTy != MVT::v4i32)
7039 return 0;
7040 if ((Subtarget->hasNEON())) {
7041 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7042 }
7043 return 0;
7044}
7045
7046unsigned fastEmit_ARMISD_VQSHLuIMM_MVT_v1i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7047 if (RetVT.SimpleTy != MVT::v1i64)
7048 return 0;
7049 if ((Subtarget->hasNEON())) {
7050 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7051 }
7052 return 0;
7053}
7054
7055unsigned fastEmit_ARMISD_VQSHLuIMM_MVT_v2i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7056 if (RetVT.SimpleTy != MVT::v2i64)
7057 return 0;
7058 if ((Subtarget->hasNEON())) {
7059 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7060 }
7061 return 0;
7062}
7063
7064unsigned fastEmit_ARMISD_VQSHLuIMM_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7065 switch (VT.SimpleTy) {
7066 case MVT::v8i8: return fastEmit_ARMISD_VQSHLuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7067 case MVT::v16i8: return fastEmit_ARMISD_VQSHLuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7068 case MVT::v4i16: return fastEmit_ARMISD_VQSHLuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7069 case MVT::v8i16: return fastEmit_ARMISD_VQSHLuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7070 case MVT::v2i32: return fastEmit_ARMISD_VQSHLuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7071 case MVT::v4i32: return fastEmit_ARMISD_VQSHLuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7072 case MVT::v1i64: return fastEmit_ARMISD_VQSHLuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7073 case MVT::v2i64: return fastEmit_ARMISD_VQSHLuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7074 default: return 0;
7075 }
7076}
7077
7078// FastEmit functions for ARMISD::VRSHRsIMM.
7079
7080unsigned fastEmit_ARMISD_VRSHRsIMM_MVT_v8i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7081 if (RetVT.SimpleTy != MVT::v8i8)
7082 return 0;
7083 if ((Subtarget->hasNEON())) {
7084 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7085 }
7086 return 0;
7087}
7088
7089unsigned fastEmit_ARMISD_VRSHRsIMM_MVT_v16i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7090 if (RetVT.SimpleTy != MVT::v16i8)
7091 return 0;
7092 if ((Subtarget->hasNEON())) {
7093 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7094 }
7095 return 0;
7096}
7097
7098unsigned fastEmit_ARMISD_VRSHRsIMM_MVT_v4i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7099 if (RetVT.SimpleTy != MVT::v4i16)
7100 return 0;
7101 if ((Subtarget->hasNEON())) {
7102 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7103 }
7104 return 0;
7105}
7106
7107unsigned fastEmit_ARMISD_VRSHRsIMM_MVT_v8i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7108 if (RetVT.SimpleTy != MVT::v8i16)
7109 return 0;
7110 if ((Subtarget->hasNEON())) {
7111 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7112 }
7113 return 0;
7114}
7115
7116unsigned fastEmit_ARMISD_VRSHRsIMM_MVT_v2i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7117 if (RetVT.SimpleTy != MVT::v2i32)
7118 return 0;
7119 if ((Subtarget->hasNEON())) {
7120 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7121 }
7122 return 0;
7123}
7124
7125unsigned fastEmit_ARMISD_VRSHRsIMM_MVT_v4i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7126 if (RetVT.SimpleTy != MVT::v4i32)
7127 return 0;
7128 if ((Subtarget->hasNEON())) {
7129 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7130 }
7131 return 0;
7132}
7133
7134unsigned fastEmit_ARMISD_VRSHRsIMM_MVT_v1i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7135 if (RetVT.SimpleTy != MVT::v1i64)
7136 return 0;
7137 if ((Subtarget->hasNEON())) {
7138 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7139 }
7140 return 0;
7141}
7142
7143unsigned fastEmit_ARMISD_VRSHRsIMM_MVT_v2i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7144 if (RetVT.SimpleTy != MVT::v2i64)
7145 return 0;
7146 if ((Subtarget->hasNEON())) {
7147 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7148 }
7149 return 0;
7150}
7151
7152unsigned fastEmit_ARMISD_VRSHRsIMM_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7153 switch (VT.SimpleTy) {
7154 case MVT::v8i8: return fastEmit_ARMISD_VRSHRsIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7155 case MVT::v16i8: return fastEmit_ARMISD_VRSHRsIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7156 case MVT::v4i16: return fastEmit_ARMISD_VRSHRsIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7157 case MVT::v8i16: return fastEmit_ARMISD_VRSHRsIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7158 case MVT::v2i32: return fastEmit_ARMISD_VRSHRsIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7159 case MVT::v4i32: return fastEmit_ARMISD_VRSHRsIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7160 case MVT::v1i64: return fastEmit_ARMISD_VRSHRsIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7161 case MVT::v2i64: return fastEmit_ARMISD_VRSHRsIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7162 default: return 0;
7163 }
7164}
7165
7166// FastEmit functions for ARMISD::VRSHRuIMM.
7167
7168unsigned fastEmit_ARMISD_VRSHRuIMM_MVT_v8i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7169 if (RetVT.SimpleTy != MVT::v8i8)
7170 return 0;
7171 if ((Subtarget->hasNEON())) {
7172 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7173 }
7174 return 0;
7175}
7176
7177unsigned fastEmit_ARMISD_VRSHRuIMM_MVT_v16i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7178 if (RetVT.SimpleTy != MVT::v16i8)
7179 return 0;
7180 if ((Subtarget->hasNEON())) {
7181 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7182 }
7183 return 0;
7184}
7185
7186unsigned fastEmit_ARMISD_VRSHRuIMM_MVT_v4i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7187 if (RetVT.SimpleTy != MVT::v4i16)
7188 return 0;
7189 if ((Subtarget->hasNEON())) {
7190 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7191 }
7192 return 0;
7193}
7194
7195unsigned fastEmit_ARMISD_VRSHRuIMM_MVT_v8i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7196 if (RetVT.SimpleTy != MVT::v8i16)
7197 return 0;
7198 if ((Subtarget->hasNEON())) {
7199 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7200 }
7201 return 0;
7202}
7203
7204unsigned fastEmit_ARMISD_VRSHRuIMM_MVT_v2i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7205 if (RetVT.SimpleTy != MVT::v2i32)
7206 return 0;
7207 if ((Subtarget->hasNEON())) {
7208 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7209 }
7210 return 0;
7211}
7212
7213unsigned fastEmit_ARMISD_VRSHRuIMM_MVT_v4i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7214 if (RetVT.SimpleTy != MVT::v4i32)
7215 return 0;
7216 if ((Subtarget->hasNEON())) {
7217 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7218 }
7219 return 0;
7220}
7221
7222unsigned fastEmit_ARMISD_VRSHRuIMM_MVT_v1i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7223 if (RetVT.SimpleTy != MVT::v1i64)
7224 return 0;
7225 if ((Subtarget->hasNEON())) {
7226 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7227 }
7228 return 0;
7229}
7230
7231unsigned fastEmit_ARMISD_VRSHRuIMM_MVT_v2i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7232 if (RetVT.SimpleTy != MVT::v2i64)
7233 return 0;
7234 if ((Subtarget->hasNEON())) {
7235 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7236 }
7237 return 0;
7238}
7239
7240unsigned fastEmit_ARMISD_VRSHRuIMM_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7241 switch (VT.SimpleTy) {
7242 case MVT::v8i8: return fastEmit_ARMISD_VRSHRuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7243 case MVT::v16i8: return fastEmit_ARMISD_VRSHRuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7244 case MVT::v4i16: return fastEmit_ARMISD_VRSHRuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7245 case MVT::v8i16: return fastEmit_ARMISD_VRSHRuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7246 case MVT::v2i32: return fastEmit_ARMISD_VRSHRuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7247 case MVT::v4i32: return fastEmit_ARMISD_VRSHRuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7248 case MVT::v1i64: return fastEmit_ARMISD_VRSHRuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7249 case MVT::v2i64: return fastEmit_ARMISD_VRSHRuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7250 default: return 0;
7251 }
7252}
7253
7254// FastEmit functions for ARMISD::VSHLIMM.
7255
7256unsigned fastEmit_ARMISD_VSHLIMM_MVT_v8i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7257 if (RetVT.SimpleTy != MVT::v8i8)
7258 return 0;
7259 if ((Subtarget->hasNEON())) {
7260 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7261 }
7262 return 0;
7263}
7264
7265unsigned fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7266 if (RetVT.SimpleTy != MVT::v16i8)
7267 return 0;
7268 if ((Subtarget->hasNEON())) {
7269 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7270 }
7271 return 0;
7272}
7273
7274unsigned fastEmit_ARMISD_VSHLIMM_MVT_v4i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7275 if (RetVT.SimpleTy != MVT::v4i16)
7276 return 0;
7277 if ((Subtarget->hasNEON())) {
7278 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7279 }
7280 return 0;
7281}
7282
7283unsigned fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7284 if (RetVT.SimpleTy != MVT::v8i16)
7285 return 0;
7286 if ((Subtarget->hasNEON())) {
7287 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7288 }
7289 return 0;
7290}
7291
7292unsigned fastEmit_ARMISD_VSHLIMM_MVT_v2i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7293 if (RetVT.SimpleTy != MVT::v2i32)
7294 return 0;
7295 if ((Subtarget->hasNEON())) {
7296 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7297 }
7298 return 0;
7299}
7300
7301unsigned fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7302 if (RetVT.SimpleTy != MVT::v4i32)
7303 return 0;
7304 if ((Subtarget->hasNEON())) {
7305 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7306 }
7307 return 0;
7308}
7309
7310unsigned fastEmit_ARMISD_VSHLIMM_MVT_v1i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7311 if (RetVT.SimpleTy != MVT::v1i64)
7312 return 0;
7313 if ((Subtarget->hasNEON())) {
7314 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7315 }
7316 return 0;
7317}
7318
7319unsigned fastEmit_ARMISD_VSHLIMM_MVT_v2i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7320 if (RetVT.SimpleTy != MVT::v2i64)
7321 return 0;
7322 if ((Subtarget->hasNEON())) {
7323 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7324 }
7325 return 0;
7326}
7327
7328unsigned fastEmit_ARMISD_VSHLIMM_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7329 switch (VT.SimpleTy) {
7330 case MVT::v8i8: return fastEmit_ARMISD_VSHLIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7331 case MVT::v16i8: return fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7332 case MVT::v4i16: return fastEmit_ARMISD_VSHLIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7333 case MVT::v8i16: return fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7334 case MVT::v2i32: return fastEmit_ARMISD_VSHLIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7335 case MVT::v4i32: return fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7336 case MVT::v1i64: return fastEmit_ARMISD_VSHLIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7337 case MVT::v2i64: return fastEmit_ARMISD_VSHLIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7338 default: return 0;
7339 }
7340}
7341
7342// FastEmit functions for ARMISD::VSHRsIMM.
7343
7344unsigned fastEmit_ARMISD_VSHRsIMM_MVT_v8i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7345 if (RetVT.SimpleTy != MVT::v8i8)
7346 return 0;
7347 if ((Subtarget->hasNEON())) {
7348 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7349 }
7350 return 0;
7351}
7352
7353unsigned fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7354 if (RetVT.SimpleTy != MVT::v16i8)
7355 return 0;
7356 if ((Subtarget->hasNEON())) {
7357 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7358 }
7359 return 0;
7360}
7361
7362unsigned fastEmit_ARMISD_VSHRsIMM_MVT_v4i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7363 if (RetVT.SimpleTy != MVT::v4i16)
7364 return 0;
7365 if ((Subtarget->hasNEON())) {
7366 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7367 }
7368 return 0;
7369}
7370
7371unsigned fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7372 if (RetVT.SimpleTy != MVT::v8i16)
7373 return 0;
7374 if ((Subtarget->hasNEON())) {
7375 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7376 }
7377 return 0;
7378}
7379
7380unsigned fastEmit_ARMISD_VSHRsIMM_MVT_v2i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7381 if (RetVT.SimpleTy != MVT::v2i32)
7382 return 0;
7383 if ((Subtarget->hasNEON())) {
7384 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7385 }
7386 return 0;
7387}
7388
7389unsigned fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7390 if (RetVT.SimpleTy != MVT::v4i32)
7391 return 0;
7392 if ((Subtarget->hasNEON())) {
7393 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7394 }
7395 return 0;
7396}
7397
7398unsigned fastEmit_ARMISD_VSHRsIMM_MVT_v1i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7399 if (RetVT.SimpleTy != MVT::v1i64)
7400 return 0;
7401 if ((Subtarget->hasNEON())) {
7402 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7403 }
7404 return 0;
7405}
7406
7407unsigned fastEmit_ARMISD_VSHRsIMM_MVT_v2i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7408 if (RetVT.SimpleTy != MVT::v2i64)
7409 return 0;
7410 if ((Subtarget->hasNEON())) {
7411 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7412 }
7413 return 0;
7414}
7415
7416unsigned fastEmit_ARMISD_VSHRsIMM_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7417 switch (VT.SimpleTy) {
7418 case MVT::v8i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7419 case MVT::v16i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7420 case MVT::v4i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7421 case MVT::v8i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7422 case MVT::v2i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7423 case MVT::v4i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7424 case MVT::v1i64: return fastEmit_ARMISD_VSHRsIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7425 case MVT::v2i64: return fastEmit_ARMISD_VSHRsIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7426 default: return 0;
7427 }
7428}
7429
7430// FastEmit functions for ARMISD::VSHRuIMM.
7431
7432unsigned fastEmit_ARMISD_VSHRuIMM_MVT_v8i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7433 if (RetVT.SimpleTy != MVT::v8i8)
7434 return 0;
7435 if ((Subtarget->hasNEON())) {
7436 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7437 }
7438 return 0;
7439}
7440
7441unsigned fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7442 if (RetVT.SimpleTy != MVT::v16i8)
7443 return 0;
7444 if ((Subtarget->hasNEON())) {
7445 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7446 }
7447 return 0;
7448}
7449
7450unsigned fastEmit_ARMISD_VSHRuIMM_MVT_v4i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7451 if (RetVT.SimpleTy != MVT::v4i16)
7452 return 0;
7453 if ((Subtarget->hasNEON())) {
7454 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7455 }
7456 return 0;
7457}
7458
7459unsigned fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7460 if (RetVT.SimpleTy != MVT::v8i16)
7461 return 0;
7462 if ((Subtarget->hasNEON())) {
7463 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7464 }
7465 return 0;
7466}
7467
7468unsigned fastEmit_ARMISD_VSHRuIMM_MVT_v2i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7469 if (RetVT.SimpleTy != MVT::v2i32)
7470 return 0;
7471 if ((Subtarget->hasNEON())) {
7472 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7473 }
7474 return 0;
7475}
7476
7477unsigned fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7478 if (RetVT.SimpleTy != MVT::v4i32)
7479 return 0;
7480 if ((Subtarget->hasNEON())) {
7481 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7482 }
7483 return 0;
7484}
7485
7486unsigned fastEmit_ARMISD_VSHRuIMM_MVT_v1i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7487 if (RetVT.SimpleTy != MVT::v1i64)
7488 return 0;
7489 if ((Subtarget->hasNEON())) {
7490 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7491 }
7492 return 0;
7493}
7494
7495unsigned fastEmit_ARMISD_VSHRuIMM_MVT_v2i64_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7496 if (RetVT.SimpleTy != MVT::v2i64)
7497 return 0;
7498 if ((Subtarget->hasNEON())) {
7499 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7500 }
7501 return 0;
7502}
7503
7504unsigned fastEmit_ARMISD_VSHRuIMM_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7505 switch (VT.SimpleTy) {
7506 case MVT::v8i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
7507 case MVT::v16i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
7508 case MVT::v4i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
7509 case MVT::v8i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
7510 case MVT::v2i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
7511 case MVT::v4i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
7512 case MVT::v1i64: return fastEmit_ARMISD_VSHRuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
7513 case MVT::v2i64: return fastEmit_ARMISD_VSHRuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
7514 default: return 0;
7515 }
7516}
7517
7518// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
7519
7520unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7521 if (RetVT.SimpleTy != MVT::i32)
7522 return 0;
7523 if ((Subtarget->hasFPRegs()) && (!Subtarget->hasSlowVGETLNi32())) {
7524 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNi32, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7525 }
7526 return 0;
7527}
7528
7529unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7530 switch (VT.SimpleTy) {
7531 case MVT::v2i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i32_ri(RetVT, Op0, imm1);
7532 default: return 0;
7533 }
7534}
7535
7536// FastEmit functions for ISD::SHL.
7537
7538unsigned fastEmit_ISD_SHL_MVT_i32_ri(MVT RetVT, unsigned Op0, uint64_t imm1) {
7539 if (RetVT.SimpleTy != MVT::i32)
7540 return 0;
7541 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
7542 return fastEmitInst_ri(MachineInstOpcode: ARM::tLSLri, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
7543 }
7544 return 0;
7545}
7546
7547unsigned fastEmit_ISD_SHL_ri(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7548 switch (VT.SimpleTy) {
7549 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri(RetVT, Op0, imm1);
7550 default: return 0;
7551 }
7552}
7553
7554// Top-level FastEmit function.
7555
7556unsigned fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) override {
7557 if (VT == MVT::i32 && Predicate_mod_imm(Imm: imm1))
7558 if (unsigned Reg = fastEmit_ri_Predicate_mod_imm(VT, RetVT, Opcode, Op0, imm1))
7559 return Reg;
7560
7561 if (VT == MVT::i32 && Predicate_imm0_7(Imm: imm1))
7562 if (unsigned Reg = fastEmit_ri_Predicate_imm0_7(VT, RetVT, Opcode, Op0, imm1))
7563 return Reg;
7564
7565 if (VT == MVT::i32 && Predicate_imm0_255_expr(Imm: imm1))
7566 if (unsigned Reg = fastEmit_ri_Predicate_imm0_255_expr(VT, RetVT, Opcode, Op0, imm1))
7567 return Reg;
7568
7569 if (VT == MVT::i32 && Predicate_imm0_255(Imm: imm1))
7570 if (unsigned Reg = fastEmit_ri_Predicate_imm0_255(VT, RetVT, Opcode, Op0, imm1))
7571 return Reg;
7572
7573 if (VT == MVT::i32 && Predicate_t2_so_imm(Imm: imm1))
7574 if (unsigned Reg = fastEmit_ri_Predicate_t2_so_imm(VT, RetVT, Opcode, Op0, imm1))
7575 return Reg;
7576
7577 if (VT == MVT::i32 && Predicate_imm0_4095(Imm: imm1))
7578 if (unsigned Reg = fastEmit_ri_Predicate_imm0_4095(VT, RetVT, Opcode, Op0, imm1))
7579 return Reg;
7580
7581 if (VT == MVT::i32 && Predicate_imm1_31(Imm: imm1))
7582 if (unsigned Reg = fastEmit_ri_Predicate_imm1_31(VT, RetVT, Opcode, Op0, imm1))
7583 return Reg;
7584
7585 if (VT == MVT::i32 && Predicate_shr_imm8(Imm: imm1))
7586 if (unsigned Reg = fastEmit_ri_Predicate_shr_imm8(VT, RetVT, Opcode, Op0, imm1))
7587 return Reg;
7588
7589 if (VT == MVT::i32 && Predicate_shr_imm16(Imm: imm1))
7590 if (unsigned Reg = fastEmit_ri_Predicate_shr_imm16(VT, RetVT, Opcode, Op0, imm1))
7591 return Reg;
7592
7593 if (VT == MVT::i32 && Predicate_shr_imm32(Imm: imm1))
7594 if (unsigned Reg = fastEmit_ri_Predicate_shr_imm32(VT, RetVT, Opcode, Op0, imm1))
7595 return Reg;
7596
7597 if (VT == MVT::i32 && Predicate_VectorIndex32(Imm: imm1))
7598 if (unsigned Reg = fastEmit_ri_Predicate_VectorIndex32(VT, RetVT, Opcode, Op0, imm1))
7599 return Reg;
7600
7601 if (VT == MVT::i32 && Predicate_imm0_31(Imm: imm1))
7602 if (unsigned Reg = fastEmit_ri_Predicate_imm0_31(VT, RetVT, Opcode, Op0, imm1))
7603 return Reg;
7604
7605 if (VT == MVT::i32 && Predicate_imm0_15(Imm: imm1))
7606 if (unsigned Reg = fastEmit_ri_Predicate_imm0_15(VT, RetVT, Opcode, Op0, imm1))
7607 return Reg;
7608
7609 switch (Opcode) {
7610 case ARMISD::PIC_ADD: return fastEmit_ARMISD_PIC_ADD_ri(VT, RetVT, Op0, imm1);
7611 case ARMISD::VDUPLANE: return fastEmit_ARMISD_VDUPLANE_ri(VT, RetVT, Op0, imm1);
7612 case ARMISD::VGETLANEs: return fastEmit_ARMISD_VGETLANEs_ri(VT, RetVT, Op0, imm1);
7613 case ARMISD::VGETLANEu: return fastEmit_ARMISD_VGETLANEu_ri(VT, RetVT, Op0, imm1);
7614 case ARMISD::VQSHLsIMM: return fastEmit_ARMISD_VQSHLsIMM_ri(VT, RetVT, Op0, imm1);
7615 case ARMISD::VQSHLsuIMM: return fastEmit_ARMISD_VQSHLsuIMM_ri(VT, RetVT, Op0, imm1);
7616 case ARMISD::VQSHLuIMM: return fastEmit_ARMISD_VQSHLuIMM_ri(VT, RetVT, Op0, imm1);
7617 case ARMISD::VRSHRsIMM: return fastEmit_ARMISD_VRSHRsIMM_ri(VT, RetVT, Op0, imm1);
7618 case ARMISD::VRSHRuIMM: return fastEmit_ARMISD_VRSHRuIMM_ri(VT, RetVT, Op0, imm1);
7619 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri(VT, RetVT, Op0, imm1);
7620 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri(VT, RetVT, Op0, imm1);
7621 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri(VT, RetVT, Op0, imm1);
7622 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(VT, RetVT, Op0, imm1);
7623 case ISD::SHL: return fastEmit_ISD_SHL_ri(VT, RetVT, Op0, imm1);
7624 default: return 0;
7625 }
7626}
7627
7628// FastEmit functions for ARMISD::CMN.
7629
7630unsigned fastEmit_ARMISD_CMN_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, unsigned Op0, uint64_t imm1) {
7631 if (RetVT.SimpleTy != MVT::isVoid)
7632 return 0;
7633 if ((!Subtarget->isThumb())) {
7634 return fastEmitInst_ri(MachineInstOpcode: ARM::CMNri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7635 }
7636 return 0;
7637}
7638
7639unsigned fastEmit_ARMISD_CMN_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7640 switch (VT.SimpleTy) {
7641 case MVT::i32: return fastEmit_ARMISD_CMN_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
7642 default: return 0;
7643 }
7644}
7645
7646// FastEmit functions for ARMISD::CMP.
7647
7648unsigned fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, unsigned Op0, uint64_t imm1) {
7649 if (RetVT.SimpleTy != MVT::isVoid)
7650 return 0;
7651 if ((!Subtarget->isThumb())) {
7652 return fastEmitInst_ri(MachineInstOpcode: ARM::CMPri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7653 }
7654 return 0;
7655}
7656
7657unsigned fastEmit_ARMISD_CMP_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7658 switch (VT.SimpleTy) {
7659 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
7660 default: return 0;
7661 }
7662}
7663
7664// FastEmit functions for ARMISD::CMPZ.
7665
7666unsigned fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, unsigned Op0, uint64_t imm1) {
7667 if (RetVT.SimpleTy != MVT::isVoid)
7668 return 0;
7669 if ((!Subtarget->isThumb())) {
7670 return fastEmitInst_ri(MachineInstOpcode: ARM::CMPri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7671 }
7672 return 0;
7673}
7674
7675unsigned fastEmit_ARMISD_CMPZ_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7676 switch (VT.SimpleTy) {
7677 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
7678 default: return 0;
7679 }
7680}
7681
7682// FastEmit functions for ISD::ADD.
7683
7684unsigned fastEmit_ISD_ADD_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, unsigned Op0, uint64_t imm1) {
7685 if (RetVT.SimpleTy != MVT::i32)
7686 return 0;
7687 if ((!Subtarget->isThumb())) {
7688 return fastEmitInst_ri(MachineInstOpcode: ARM::ADDri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7689 }
7690 return 0;
7691}
7692
7693unsigned fastEmit_ISD_ADD_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7694 switch (VT.SimpleTy) {
7695 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
7696 default: return 0;
7697 }
7698}
7699
7700// FastEmit functions for ISD::AND.
7701
7702unsigned fastEmit_ISD_AND_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, unsigned Op0, uint64_t imm1) {
7703 if (RetVT.SimpleTy != MVT::i32)
7704 return 0;
7705 if ((!Subtarget->isThumb())) {
7706 return fastEmitInst_ri(MachineInstOpcode: ARM::ANDri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7707 }
7708 return 0;
7709}
7710
7711unsigned fastEmit_ISD_AND_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7712 switch (VT.SimpleTy) {
7713 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
7714 default: return 0;
7715 }
7716}
7717
7718// FastEmit functions for ISD::OR.
7719
7720unsigned fastEmit_ISD_OR_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, unsigned Op0, uint64_t imm1) {
7721 if (RetVT.SimpleTy != MVT::i32)
7722 return 0;
7723 if ((!Subtarget->isThumb())) {
7724 return fastEmitInst_ri(MachineInstOpcode: ARM::ORRri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7725 }
7726 return 0;
7727}
7728
7729unsigned fastEmit_ISD_OR_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7730 switch (VT.SimpleTy) {
7731 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
7732 default: return 0;
7733 }
7734}
7735
7736// FastEmit functions for ISD::SUB.
7737
7738unsigned fastEmit_ISD_SUB_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, unsigned Op0, uint64_t imm1) {
7739 if (RetVT.SimpleTy != MVT::i32)
7740 return 0;
7741 if ((!Subtarget->isThumb())) {
7742 return fastEmitInst_ri(MachineInstOpcode: ARM::SUBri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7743 }
7744 return 0;
7745}
7746
7747unsigned fastEmit_ISD_SUB_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7748 switch (VT.SimpleTy) {
7749 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
7750 default: return 0;
7751 }
7752}
7753
7754// FastEmit functions for ISD::XOR.
7755
7756unsigned fastEmit_ISD_XOR_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, unsigned Op0, uint64_t imm1) {
7757 if (RetVT.SimpleTy != MVT::i32)
7758 return 0;
7759 if ((!Subtarget->isThumb())) {
7760 return fastEmitInst_ri(MachineInstOpcode: ARM::EORri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7761 }
7762 return 0;
7763}
7764
7765unsigned fastEmit_ISD_XOR_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7766 switch (VT.SimpleTy) {
7767 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
7768 default: return 0;
7769 }
7770}
7771
7772// Top-level FastEmit function.
7773
7774unsigned fastEmit_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
7775 switch (Opcode) {
7776 case ARMISD::CMN: return fastEmit_ARMISD_CMN_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
7777 case ARMISD::CMP: return fastEmit_ARMISD_CMP_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
7778 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
7779 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
7780 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
7781 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
7782 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
7783 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
7784 default: return 0;
7785 }
7786}
7787
7788// FastEmit functions for ARMISD::VSHLIMM.
7789
7790unsigned fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, unsigned Op0, uint64_t imm1) {
7791 if (RetVT.SimpleTy != MVT::v16i8)
7792 return 0;
7793 if ((Subtarget->hasMVEIntegerOps())) {
7794 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHL_immi8, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
7795 }
7796 return 0;
7797}
7798
7799unsigned fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7800 switch (VT.SimpleTy) {
7801 case MVT::v16i8: return fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, imm1);
7802 default: return 0;
7803 }
7804}
7805
7806// FastEmit functions for ARMISD::VSHRsIMM.
7807
7808unsigned fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, unsigned Op0, uint64_t imm1) {
7809 if (RetVT.SimpleTy != MVT::v16i8)
7810 return 0;
7811 if ((Subtarget->hasMVEIntegerOps())) {
7812 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_imms8, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
7813 }
7814 return 0;
7815}
7816
7817unsigned fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7818 switch (VT.SimpleTy) {
7819 case MVT::v16i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, imm1);
7820 default: return 0;
7821 }
7822}
7823
7824// FastEmit functions for ARMISD::VSHRuIMM.
7825
7826unsigned fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, unsigned Op0, uint64_t imm1) {
7827 if (RetVT.SimpleTy != MVT::v16i8)
7828 return 0;
7829 if ((Subtarget->hasMVEIntegerOps())) {
7830 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_immu8, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
7831 }
7832 return 0;
7833}
7834
7835unsigned fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7836 switch (VT.SimpleTy) {
7837 case MVT::v16i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, imm1);
7838 default: return 0;
7839 }
7840}
7841
7842// FastEmit functions for ISD::ADD.
7843
7844unsigned fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_7(MVT RetVT, unsigned Op0, uint64_t imm1) {
7845 if (RetVT.SimpleTy != MVT::i32)
7846 return 0;
7847 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
7848 return fastEmitInst_ri(MachineInstOpcode: ARM::tADDi3, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
7849 }
7850 return 0;
7851}
7852
7853unsigned fastEmit_ISD_ADD_ri_Predicate_imm0_7(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7854 switch (VT.SimpleTy) {
7855 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_7(RetVT, Op0, imm1);
7856 default: return 0;
7857 }
7858}
7859
7860// Top-level FastEmit function.
7861
7862unsigned fastEmit_ri_Predicate_imm0_7(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
7863 switch (Opcode) {
7864 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
7865 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
7866 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
7867 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
7868 default: return 0;
7869 }
7870}
7871
7872// FastEmit functions for ISD::ADD.
7873
7874unsigned fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_255_expr(MVT RetVT, unsigned Op0, uint64_t imm1) {
7875 if (RetVT.SimpleTy != MVT::i32)
7876 return 0;
7877 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
7878 return fastEmitInst_ri(MachineInstOpcode: ARM::tADDi8, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
7879 }
7880 return 0;
7881}
7882
7883unsigned fastEmit_ISD_ADD_ri_Predicate_imm0_255_expr(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7884 switch (VT.SimpleTy) {
7885 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_255_expr(RetVT, Op0, imm1);
7886 default: return 0;
7887 }
7888}
7889
7890// Top-level FastEmit function.
7891
7892unsigned fastEmit_ri_Predicate_imm0_255_expr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
7893 switch (Opcode) {
7894 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm0_255_expr(VT, RetVT, Op0, imm1);
7895 default: return 0;
7896 }
7897}
7898
7899// FastEmit functions for ARMISD::CMP.
7900
7901unsigned fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_imm0_255(MVT RetVT, unsigned Op0, uint64_t imm1) {
7902 if (RetVT.SimpleTy != MVT::isVoid)
7903 return 0;
7904 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
7905 return fastEmitInst_ri(MachineInstOpcode: ARM::tCMPi8, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
7906 }
7907 return 0;
7908}
7909
7910unsigned fastEmit_ARMISD_CMP_ri_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7911 switch (VT.SimpleTy) {
7912 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_imm0_255(RetVT, Op0, imm1);
7913 default: return 0;
7914 }
7915}
7916
7917// FastEmit functions for ARMISD::CMPZ.
7918
7919unsigned fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_imm0_255(MVT RetVT, unsigned Op0, uint64_t imm1) {
7920 if (RetVT.SimpleTy != MVT::isVoid)
7921 return 0;
7922 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
7923 return fastEmitInst_ri(MachineInstOpcode: ARM::tCMPi8, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
7924 }
7925 return 0;
7926}
7927
7928unsigned fastEmit_ARMISD_CMPZ_ri_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7929 switch (VT.SimpleTy) {
7930 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_imm0_255(RetVT, Op0, imm1);
7931 default: return 0;
7932 }
7933}
7934
7935// Top-level FastEmit function.
7936
7937unsigned fastEmit_ri_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
7938 switch (Opcode) {
7939 case ARMISD::CMP: return fastEmit_ARMISD_CMP_ri_Predicate_imm0_255(VT, RetVT, Op0, imm1);
7940 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_ri_Predicate_imm0_255(VT, RetVT, Op0, imm1);
7941 default: return 0;
7942 }
7943}
7944
7945// FastEmit functions for ARMISD::CMP.
7946
7947unsigned fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, unsigned Op0, uint64_t imm1) {
7948 if (RetVT.SimpleTy != MVT::isVoid)
7949 return 0;
7950 if ((Subtarget->isThumb2())) {
7951 return fastEmitInst_ri(MachineInstOpcode: ARM::t2CMPri, RC: &ARM::GPRnopcRegClass, Op0, Imm: imm1);
7952 }
7953 return 0;
7954}
7955
7956unsigned fastEmit_ARMISD_CMP_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7957 switch (VT.SimpleTy) {
7958 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
7959 default: return 0;
7960 }
7961}
7962
7963// FastEmit functions for ARMISD::CMPZ.
7964
7965unsigned fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, unsigned Op0, uint64_t imm1) {
7966 if (RetVT.SimpleTy != MVT::isVoid)
7967 return 0;
7968 if ((Subtarget->isThumb2())) {
7969 return fastEmitInst_ri(MachineInstOpcode: ARM::t2CMPri, RC: &ARM::GPRnopcRegClass, Op0, Imm: imm1);
7970 }
7971 return 0;
7972}
7973
7974unsigned fastEmit_ARMISD_CMPZ_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7975 switch (VT.SimpleTy) {
7976 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
7977 default: return 0;
7978 }
7979}
7980
7981// FastEmit functions for ISD::ADD.
7982
7983unsigned fastEmit_ISD_ADD_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, unsigned Op0, uint64_t imm1) {
7984 if (RetVT.SimpleTy != MVT::i32)
7985 return 0;
7986 if ((Subtarget->isThumb2())) {
7987 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ADDri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7988 }
7989 return 0;
7990}
7991
7992unsigned fastEmit_ISD_ADD_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
7993 switch (VT.SimpleTy) {
7994 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
7995 default: return 0;
7996 }
7997}
7998
7999// FastEmit functions for ISD::AND.
8000
8001unsigned fastEmit_ISD_AND_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, unsigned Op0, uint64_t imm1) {
8002 if (RetVT.SimpleTy != MVT::i32)
8003 return 0;
8004 if ((Subtarget->isThumb2())) {
8005 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ANDri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8006 }
8007 return 0;
8008}
8009
8010unsigned fastEmit_ISD_AND_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8011 switch (VT.SimpleTy) {
8012 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8013 default: return 0;
8014 }
8015}
8016
8017// FastEmit functions for ISD::OR.
8018
8019unsigned fastEmit_ISD_OR_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, unsigned Op0, uint64_t imm1) {
8020 if (RetVT.SimpleTy != MVT::i32)
8021 return 0;
8022 if ((Subtarget->isThumb2())) {
8023 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ORRri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8024 }
8025 return 0;
8026}
8027
8028unsigned fastEmit_ISD_OR_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8029 switch (VT.SimpleTy) {
8030 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8031 default: return 0;
8032 }
8033}
8034
8035// FastEmit functions for ISD::SUB.
8036
8037unsigned fastEmit_ISD_SUB_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, unsigned Op0, uint64_t imm1) {
8038 if (RetVT.SimpleTy != MVT::i32)
8039 return 0;
8040 if ((Subtarget->isThumb2())) {
8041 return fastEmitInst_ri(MachineInstOpcode: ARM::t2SUBri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8042 }
8043 return 0;
8044}
8045
8046unsigned fastEmit_ISD_SUB_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8047 switch (VT.SimpleTy) {
8048 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8049 default: return 0;
8050 }
8051}
8052
8053// FastEmit functions for ISD::XOR.
8054
8055unsigned fastEmit_ISD_XOR_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, unsigned Op0, uint64_t imm1) {
8056 if (RetVT.SimpleTy != MVT::i32)
8057 return 0;
8058 if ((Subtarget->isThumb2())) {
8059 return fastEmitInst_ri(MachineInstOpcode: ARM::t2EORri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8060 }
8061 return 0;
8062}
8063
8064unsigned fastEmit_ISD_XOR_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8065 switch (VT.SimpleTy) {
8066 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
8067 default: return 0;
8068 }
8069}
8070
8071// Top-level FastEmit function.
8072
8073unsigned fastEmit_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
8074 switch (Opcode) {
8075 case ARMISD::CMP: return fastEmit_ARMISD_CMP_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8076 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8077 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8078 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8079 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8080 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8081 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
8082 default: return 0;
8083 }
8084}
8085
8086// FastEmit functions for ISD::ADD.
8087
8088unsigned fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_4095(MVT RetVT, unsigned Op0, uint64_t imm1) {
8089 if (RetVT.SimpleTy != MVT::i32)
8090 return 0;
8091 if ((Subtarget->isThumb2())) {
8092 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ADDri12, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8093 }
8094 return 0;
8095}
8096
8097unsigned fastEmit_ISD_ADD_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8098 switch (VT.SimpleTy) {
8099 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_4095(RetVT, Op0, imm1);
8100 default: return 0;
8101 }
8102}
8103
8104// FastEmit functions for ISD::SUB.
8105
8106unsigned fastEmit_ISD_SUB_MVT_i32_ri_Predicate_imm0_4095(MVT RetVT, unsigned Op0, uint64_t imm1) {
8107 if (RetVT.SimpleTy != MVT::i32)
8108 return 0;
8109 if ((Subtarget->isThumb2())) {
8110 return fastEmitInst_ri(MachineInstOpcode: ARM::t2SUBri12, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8111 }
8112 return 0;
8113}
8114
8115unsigned fastEmit_ISD_SUB_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8116 switch (VT.SimpleTy) {
8117 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_imm0_4095(RetVT, Op0, imm1);
8118 default: return 0;
8119 }
8120}
8121
8122// Top-level FastEmit function.
8123
8124unsigned fastEmit_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
8125 switch (Opcode) {
8126 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm0_4095(VT, RetVT, Op0, imm1);
8127 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_imm0_4095(VT, RetVT, Op0, imm1);
8128 default: return 0;
8129 }
8130}
8131
8132// FastEmit functions for ISD::ROTR.
8133
8134unsigned fastEmit_ISD_ROTR_MVT_i32_ri_Predicate_imm1_31(MVT RetVT, unsigned Op0, uint64_t imm1) {
8135 if (RetVT.SimpleTy != MVT::i32)
8136 return 0;
8137 if ((Subtarget->isThumb2())) {
8138 return fastEmitInst_ri(MachineInstOpcode: ARM::t2RORri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8139 }
8140 return 0;
8141}
8142
8143unsigned fastEmit_ISD_ROTR_ri_Predicate_imm1_31(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8144 switch (VT.SimpleTy) {
8145 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_ri_Predicate_imm1_31(RetVT, Op0, imm1);
8146 default: return 0;
8147 }
8148}
8149
8150// FastEmit functions for ISD::SHL.
8151
8152unsigned fastEmit_ISD_SHL_MVT_i32_ri_Predicate_imm1_31(MVT RetVT, unsigned Op0, uint64_t imm1) {
8153 if (RetVT.SimpleTy != MVT::i32)
8154 return 0;
8155 if ((Subtarget->isThumb2())) {
8156 return fastEmitInst_ri(MachineInstOpcode: ARM::t2LSLri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
8157 }
8158 return 0;
8159}
8160
8161unsigned fastEmit_ISD_SHL_ri_Predicate_imm1_31(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8162 switch (VT.SimpleTy) {
8163 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri_Predicate_imm1_31(RetVT, Op0, imm1);
8164 default: return 0;
8165 }
8166}
8167
8168// Top-level FastEmit function.
8169
8170unsigned fastEmit_ri_Predicate_imm1_31(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
8171 switch (Opcode) {
8172 case ISD::ROTR: return fastEmit_ISD_ROTR_ri_Predicate_imm1_31(VT, RetVT, Op0, imm1);
8173 case ISD::SHL: return fastEmit_ISD_SHL_ri_Predicate_imm1_31(VT, RetVT, Op0, imm1);
8174 default: return 0;
8175 }
8176}
8177
8178// FastEmit functions for ARMISD::VQRSHRNsIMM.
8179
8180unsigned fastEmit_ARMISD_VQRSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, unsigned Op0, uint64_t imm1) {
8181 if (RetVT.SimpleTy != MVT::v8i8)
8182 return 0;
8183 if ((Subtarget->hasNEON())) {
8184 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8185 }
8186 return 0;
8187}
8188
8189unsigned fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8190 switch (VT.SimpleTy) {
8191 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8192 default: return 0;
8193 }
8194}
8195
8196// FastEmit functions for ARMISD::VQRSHRNsuIMM.
8197
8198unsigned fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, unsigned Op0, uint64_t imm1) {
8199 if (RetVT.SimpleTy != MVT::v8i8)
8200 return 0;
8201 if ((Subtarget->hasNEON())) {
8202 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRUNv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8203 }
8204 return 0;
8205}
8206
8207unsigned fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8208 switch (VT.SimpleTy) {
8209 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8210 default: return 0;
8211 }
8212}
8213
8214// FastEmit functions for ARMISD::VQRSHRNuIMM.
8215
8216unsigned fastEmit_ARMISD_VQRSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, unsigned Op0, uint64_t imm1) {
8217 if (RetVT.SimpleTy != MVT::v8i8)
8218 return 0;
8219 if ((Subtarget->hasNEON())) {
8220 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8221 }
8222 return 0;
8223}
8224
8225unsigned fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8226 switch (VT.SimpleTy) {
8227 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8228 default: return 0;
8229 }
8230}
8231
8232// FastEmit functions for ARMISD::VQSHRNsIMM.
8233
8234unsigned fastEmit_ARMISD_VQSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, unsigned Op0, uint64_t imm1) {
8235 if (RetVT.SimpleTy != MVT::v8i8)
8236 return 0;
8237 if ((Subtarget->hasNEON())) {
8238 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8239 }
8240 return 0;
8241}
8242
8243unsigned fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8244 switch (VT.SimpleTy) {
8245 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8246 default: return 0;
8247 }
8248}
8249
8250// FastEmit functions for ARMISD::VQSHRNsuIMM.
8251
8252unsigned fastEmit_ARMISD_VQSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, unsigned Op0, uint64_t imm1) {
8253 if (RetVT.SimpleTy != MVT::v8i8)
8254 return 0;
8255 if ((Subtarget->hasNEON())) {
8256 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRUNv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8257 }
8258 return 0;
8259}
8260
8261unsigned fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8262 switch (VT.SimpleTy) {
8263 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8264 default: return 0;
8265 }
8266}
8267
8268// FastEmit functions for ARMISD::VQSHRNuIMM.
8269
8270unsigned fastEmit_ARMISD_VQSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, unsigned Op0, uint64_t imm1) {
8271 if (RetVT.SimpleTy != MVT::v8i8)
8272 return 0;
8273 if ((Subtarget->hasNEON())) {
8274 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8275 }
8276 return 0;
8277}
8278
8279unsigned fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8280 switch (VT.SimpleTy) {
8281 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8282 default: return 0;
8283 }
8284}
8285
8286// FastEmit functions for ARMISD::VRSHRNIMM.
8287
8288unsigned fastEmit_ARMISD_VRSHRNIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, unsigned Op0, uint64_t imm1) {
8289 if (RetVT.SimpleTy != MVT::v8i8)
8290 return 0;
8291 if ((Subtarget->hasNEON())) {
8292 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRNv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8293 }
8294 return 0;
8295}
8296
8297unsigned fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8298 switch (VT.SimpleTy) {
8299 case MVT::v8i16: return fastEmit_ARMISD_VRSHRNIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
8300 default: return 0;
8301 }
8302}
8303
8304// Top-level FastEmit function.
8305
8306unsigned fastEmit_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
8307 switch (Opcode) {
8308 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8309 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8310 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8311 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8312 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8313 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8314 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
8315 default: return 0;
8316 }
8317}
8318
8319// FastEmit functions for ARMISD::VQRSHRNsIMM.
8320
8321unsigned fastEmit_ARMISD_VQRSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, unsigned Op0, uint64_t imm1) {
8322 if (RetVT.SimpleTy != MVT::v4i16)
8323 return 0;
8324 if ((Subtarget->hasNEON())) {
8325 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8326 }
8327 return 0;
8328}
8329
8330unsigned fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8331 switch (VT.SimpleTy) {
8332 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8333 default: return 0;
8334 }
8335}
8336
8337// FastEmit functions for ARMISD::VQRSHRNsuIMM.
8338
8339unsigned fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, unsigned Op0, uint64_t imm1) {
8340 if (RetVT.SimpleTy != MVT::v4i16)
8341 return 0;
8342 if ((Subtarget->hasNEON())) {
8343 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRUNv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8344 }
8345 return 0;
8346}
8347
8348unsigned fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8349 switch (VT.SimpleTy) {
8350 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8351 default: return 0;
8352 }
8353}
8354
8355// FastEmit functions for ARMISD::VQRSHRNuIMM.
8356
8357unsigned fastEmit_ARMISD_VQRSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, unsigned Op0, uint64_t imm1) {
8358 if (RetVT.SimpleTy != MVT::v4i16)
8359 return 0;
8360 if ((Subtarget->hasNEON())) {
8361 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8362 }
8363 return 0;
8364}
8365
8366unsigned fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8367 switch (VT.SimpleTy) {
8368 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8369 default: return 0;
8370 }
8371}
8372
8373// FastEmit functions for ARMISD::VQSHRNsIMM.
8374
8375unsigned fastEmit_ARMISD_VQSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, unsigned Op0, uint64_t imm1) {
8376 if (RetVT.SimpleTy != MVT::v4i16)
8377 return 0;
8378 if ((Subtarget->hasNEON())) {
8379 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8380 }
8381 return 0;
8382}
8383
8384unsigned fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8385 switch (VT.SimpleTy) {
8386 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8387 default: return 0;
8388 }
8389}
8390
8391// FastEmit functions for ARMISD::VQSHRNsuIMM.
8392
8393unsigned fastEmit_ARMISD_VQSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, unsigned Op0, uint64_t imm1) {
8394 if (RetVT.SimpleTy != MVT::v4i16)
8395 return 0;
8396 if ((Subtarget->hasNEON())) {
8397 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRUNv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8398 }
8399 return 0;
8400}
8401
8402unsigned fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8403 switch (VT.SimpleTy) {
8404 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8405 default: return 0;
8406 }
8407}
8408
8409// FastEmit functions for ARMISD::VQSHRNuIMM.
8410
8411unsigned fastEmit_ARMISD_VQSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, unsigned Op0, uint64_t imm1) {
8412 if (RetVT.SimpleTy != MVT::v4i16)
8413 return 0;
8414 if ((Subtarget->hasNEON())) {
8415 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8416 }
8417 return 0;
8418}
8419
8420unsigned fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8421 switch (VT.SimpleTy) {
8422 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8423 default: return 0;
8424 }
8425}
8426
8427// FastEmit functions for ARMISD::VRSHRNIMM.
8428
8429unsigned fastEmit_ARMISD_VRSHRNIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, unsigned Op0, uint64_t imm1) {
8430 if (RetVT.SimpleTy != MVT::v4i16)
8431 return 0;
8432 if ((Subtarget->hasNEON())) {
8433 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRNv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8434 }
8435 return 0;
8436}
8437
8438unsigned fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8439 switch (VT.SimpleTy) {
8440 case MVT::v4i32: return fastEmit_ARMISD_VRSHRNIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
8441 default: return 0;
8442 }
8443}
8444
8445// Top-level FastEmit function.
8446
8447unsigned fastEmit_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
8448 switch (Opcode) {
8449 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
8450 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
8451 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
8452 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
8453 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
8454 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
8455 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
8456 default: return 0;
8457 }
8458}
8459
8460// FastEmit functions for ARMISD::VQRSHRNsIMM.
8461
8462unsigned fastEmit_ARMISD_VQRSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, unsigned Op0, uint64_t imm1) {
8463 if (RetVT.SimpleTy != MVT::v2i32)
8464 return 0;
8465 if ((Subtarget->hasNEON())) {
8466 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8467 }
8468 return 0;
8469}
8470
8471unsigned fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8472 switch (VT.SimpleTy) {
8473 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
8474 default: return 0;
8475 }
8476}
8477
8478// FastEmit functions for ARMISD::VQRSHRNsuIMM.
8479
8480unsigned fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, unsigned Op0, uint64_t imm1) {
8481 if (RetVT.SimpleTy != MVT::v2i32)
8482 return 0;
8483 if ((Subtarget->hasNEON())) {
8484 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRUNv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8485 }
8486 return 0;
8487}
8488
8489unsigned fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8490 switch (VT.SimpleTy) {
8491 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
8492 default: return 0;
8493 }
8494}
8495
8496// FastEmit functions for ARMISD::VQRSHRNuIMM.
8497
8498unsigned fastEmit_ARMISD_VQRSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, unsigned Op0, uint64_t imm1) {
8499 if (RetVT.SimpleTy != MVT::v2i32)
8500 return 0;
8501 if ((Subtarget->hasNEON())) {
8502 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8503 }
8504 return 0;
8505}
8506
8507unsigned fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8508 switch (VT.SimpleTy) {
8509 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
8510 default: return 0;
8511 }
8512}
8513
8514// FastEmit functions for ARMISD::VQSHRNsIMM.
8515
8516unsigned fastEmit_ARMISD_VQSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, unsigned Op0, uint64_t imm1) {
8517 if (RetVT.SimpleTy != MVT::v2i32)
8518 return 0;
8519 if ((Subtarget->hasNEON())) {
8520 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8521 }
8522 return 0;
8523}
8524
8525unsigned fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8526 switch (VT.SimpleTy) {
8527 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
8528 default: return 0;
8529 }
8530}
8531
8532// FastEmit functions for ARMISD::VQSHRNsuIMM.
8533
8534unsigned fastEmit_ARMISD_VQSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, unsigned Op0, uint64_t imm1) {
8535 if (RetVT.SimpleTy != MVT::v2i32)
8536 return 0;
8537 if ((Subtarget->hasNEON())) {
8538 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRUNv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8539 }
8540 return 0;
8541}
8542
8543unsigned fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8544 switch (VT.SimpleTy) {
8545 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
8546 default: return 0;
8547 }
8548}
8549
8550// FastEmit functions for ARMISD::VQSHRNuIMM.
8551
8552unsigned fastEmit_ARMISD_VQSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, unsigned Op0, uint64_t imm1) {
8553 if (RetVT.SimpleTy != MVT::v2i32)
8554 return 0;
8555 if ((Subtarget->hasNEON())) {
8556 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8557 }
8558 return 0;
8559}
8560
8561unsigned fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8562 switch (VT.SimpleTy) {
8563 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
8564 default: return 0;
8565 }
8566}
8567
8568// FastEmit functions for ARMISD::VRSHRNIMM.
8569
8570unsigned fastEmit_ARMISD_VRSHRNIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, unsigned Op0, uint64_t imm1) {
8571 if (RetVT.SimpleTy != MVT::v2i32)
8572 return 0;
8573 if ((Subtarget->hasNEON())) {
8574 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRNv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8575 }
8576 return 0;
8577}
8578
8579unsigned fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8580 switch (VT.SimpleTy) {
8581 case MVT::v2i64: return fastEmit_ARMISD_VRSHRNIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
8582 default: return 0;
8583 }
8584}
8585
8586// Top-level FastEmit function.
8587
8588unsigned fastEmit_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
8589 switch (Opcode) {
8590 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
8591 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
8592 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
8593 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
8594 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
8595 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
8596 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
8597 default: return 0;
8598 }
8599}
8600
8601// FastEmit functions for ARMISD::VDUPLANE.
8602
8603unsigned fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri_Predicate_VectorIndex32(MVT RetVT, unsigned Op0, uint64_t imm1) {
8604 if (RetVT.SimpleTy != MVT::v16i8)
8605 return 0;
8606 if ((Subtarget->hasNEON())) {
8607 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN8q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8608 }
8609 return 0;
8610}
8611
8612unsigned fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri_Predicate_VectorIndex32(MVT RetVT, unsigned Op0, uint64_t imm1) {
8613 if (RetVT.SimpleTy != MVT::v8i16)
8614 return 0;
8615 if ((Subtarget->hasNEON())) {
8616 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8617 }
8618 return 0;
8619}
8620
8621unsigned fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri_Predicate_VectorIndex32(MVT RetVT, unsigned Op0, uint64_t imm1) {
8622 if (RetVT.SimpleTy != MVT::v4i32)
8623 return 0;
8624 if ((Subtarget->hasNEON())) {
8625 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8626 }
8627 return 0;
8628}
8629
8630unsigned fastEmit_ARMISD_VDUPLANE_ri_Predicate_VectorIndex32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8631 switch (VT.SimpleTy) {
8632 case MVT::v8i8: return fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri_Predicate_VectorIndex32(RetVT, Op0, imm1);
8633 case MVT::v4i16: return fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri_Predicate_VectorIndex32(RetVT, Op0, imm1);
8634 case MVT::v2i32: return fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri_Predicate_VectorIndex32(RetVT, Op0, imm1);
8635 default: return 0;
8636 }
8637}
8638
8639// Top-level FastEmit function.
8640
8641unsigned fastEmit_ri_Predicate_VectorIndex32(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
8642 switch (Opcode) {
8643 case ARMISD::VDUPLANE: return fastEmit_ARMISD_VDUPLANE_ri_Predicate_VectorIndex32(VT, RetVT, Op0, imm1);
8644 default: return 0;
8645 }
8646}
8647
8648// FastEmit functions for ARMISD::VSHLIMM.
8649
8650unsigned fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, unsigned Op0, uint64_t imm1) {
8651 if (RetVT.SimpleTy != MVT::v4i32)
8652 return 0;
8653 if ((Subtarget->hasMVEIntegerOps())) {
8654 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHL_immi32, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8655 }
8656 return 0;
8657}
8658
8659unsigned fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8660 switch (VT.SimpleTy) {
8661 case MVT::v4i32: return fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, imm1);
8662 default: return 0;
8663 }
8664}
8665
8666// FastEmit functions for ARMISD::VSHRsIMM.
8667
8668unsigned fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, unsigned Op0, uint64_t imm1) {
8669 if (RetVT.SimpleTy != MVT::v4i32)
8670 return 0;
8671 if ((Subtarget->hasMVEIntegerOps())) {
8672 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_imms32, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8673 }
8674 return 0;
8675}
8676
8677unsigned fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8678 switch (VT.SimpleTy) {
8679 case MVT::v4i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, imm1);
8680 default: return 0;
8681 }
8682}
8683
8684// FastEmit functions for ARMISD::VSHRuIMM.
8685
8686unsigned fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, unsigned Op0, uint64_t imm1) {
8687 if (RetVT.SimpleTy != MVT::v4i32)
8688 return 0;
8689 if ((Subtarget->hasMVEIntegerOps())) {
8690 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_immu32, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8691 }
8692 return 0;
8693}
8694
8695unsigned fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8696 switch (VT.SimpleTy) {
8697 case MVT::v4i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, imm1);
8698 default: return 0;
8699 }
8700}
8701
8702// Top-level FastEmit function.
8703
8704unsigned fastEmit_ri_Predicate_imm0_31(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
8705 switch (Opcode) {
8706 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, imm1);
8707 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, imm1);
8708 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, imm1);
8709 default: return 0;
8710 }
8711}
8712
8713// FastEmit functions for ARMISD::VSHLIMM.
8714
8715unsigned fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, unsigned Op0, uint64_t imm1) {
8716 if (RetVT.SimpleTy != MVT::v8i16)
8717 return 0;
8718 if ((Subtarget->hasMVEIntegerOps())) {
8719 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHL_immi16, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8720 }
8721 return 0;
8722}
8723
8724unsigned fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8725 switch (VT.SimpleTy) {
8726 case MVT::v8i16: return fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, imm1);
8727 default: return 0;
8728 }
8729}
8730
8731// FastEmit functions for ARMISD::VSHRsIMM.
8732
8733unsigned fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, unsigned Op0, uint64_t imm1) {
8734 if (RetVT.SimpleTy != MVT::v8i16)
8735 return 0;
8736 if ((Subtarget->hasMVEIntegerOps())) {
8737 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_imms16, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8738 }
8739 return 0;
8740}
8741
8742unsigned fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8743 switch (VT.SimpleTy) {
8744 case MVT::v8i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, imm1);
8745 default: return 0;
8746 }
8747}
8748
8749// FastEmit functions for ARMISD::VSHRuIMM.
8750
8751unsigned fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, unsigned Op0, uint64_t imm1) {
8752 if (RetVT.SimpleTy != MVT::v8i16)
8753 return 0;
8754 if ((Subtarget->hasMVEIntegerOps())) {
8755 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_immu16, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8756 }
8757 return 0;
8758}
8759
8760unsigned fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
8761 switch (VT.SimpleTy) {
8762 case MVT::v8i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, imm1);
8763 default: return 0;
8764 }
8765}
8766
8767// Top-level FastEmit function.
8768
8769unsigned fastEmit_ri_Predicate_imm0_15(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
8770 switch (Opcode) {
8771 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, imm1);
8772 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, imm1);
8773 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, imm1);
8774 default: return 0;
8775 }
8776}
8777
8778// FastEmit functions for ISD::Constant.
8779
8780unsigned fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
8781 if (RetVT.SimpleTy != MVT::i32)
8782 return 0;
8783 if ((Subtarget->isThumb()) && (Subtarget->useMovt())) {
8784 return fastEmitInst_i(MachineInstOpcode: ARM::t2MOVi32imm, RC: &ARM::rGPRRegClass, Imm: imm0);
8785 }
8786 if ((!Subtarget->useMovt()) && (Subtarget->genExecuteOnly()) && (Subtarget->isThumb1Only())) {
8787 return fastEmitInst_i(MachineInstOpcode: ARM::tMOVi32imm, RC: &ARM::rGPRRegClass, Imm: imm0);
8788 }
8789 return 0;
8790}
8791
8792unsigned fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
8793 switch (VT.SimpleTy) {
8794 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
8795 default: return 0;
8796 }
8797}
8798
8799// Top-level FastEmit function.
8800
8801unsigned fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
8802 switch (Opcode) {
8803 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
8804 default: return 0;
8805 }
8806}
8807
8808