1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the ARM target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_mod_imm(int64_t Imm) {
12
13 return ARM_AM::getSOImmVal(Arg: Imm) != -1;
14
15}
16static bool Predicate_imm0_7(int64_t Imm) {
17
18 return Imm >= 0 && Imm < 8;
19
20}
21static bool Predicate_imm0_255_expr(int64_t Imm) {
22 return Imm >= 0 && Imm < 256;
23}
24static bool Predicate_imm_sr(int64_t Imm) {
25
26 return Imm > 0 && Imm <= 32;
27
28}
29static bool Predicate_imm0_255(int64_t Imm) {
30 return Imm >= 0 && Imm < 256;
31}
32static bool Predicate_t2_so_imm(int64_t Imm) {
33
34 return ARM_AM::getT2SOImmVal(Arg: Imm) != -1;
35
36}
37static bool Predicate_imm0_4095(int64_t Imm) {
38
39 return Imm >= 0 && Imm < 4096;
40
41}
42static bool Predicate_imm1_31(int64_t Imm) {
43 return Imm > 0 && Imm < 32;
44}
45static bool Predicate_shr_imm8(int64_t Imm) {
46 return Imm > 0 && Imm <= 8;
47}
48static bool Predicate_shr_imm16(int64_t Imm) {
49 return Imm > 0 && Imm <= 16;
50}
51static bool Predicate_shr_imm32(int64_t Imm) {
52 return Imm > 0 && Imm <= 32;
53}
54static bool Predicate_VectorIndex32(int64_t Imm) {
55
56 return ((uint64_t)Imm) < 2;
57
58}
59static bool Predicate_imm0_31(int64_t Imm) {
60
61 return Imm >= 0 && Imm < 32;
62
63}
64static bool Predicate_mod_imm_not(int64_t Imm) {
65
66 return ARM_AM::getSOImmVal(Arg: ~(uint32_t)Imm) != -1;
67
68}
69static bool Predicate_imm0_65535(int64_t Imm) {
70
71 return Imm >= 0 && Imm < 65536;
72
73}
74static bool Predicate_t2_so_imm_neg(int64_t Imm) {
75
76 return Imm && ARM_AM::getT2SOImmVal(Arg: -(uint32_t)Imm) != -1;
77
78}
79static bool Predicate_imm0_15(int64_t Imm) {
80
81 return Imm >= 0 && Imm < 16;
82
83}
84
85
86// FastEmit functions for ISD::GET_FPENV.
87
88Register fastEmit_ISD_GET_FPENV_MVT_i32_(MVT RetVT) {
89 if (RetVT.SimpleTy != MVT::i32)
90 return Register();
91 return fastEmitInst_(MachineInstOpcode: ARM::VMRS, RC: &ARM::GPRnopcRegClass);
92}
93
94Register fastEmit_ISD_GET_FPENV_(MVT VT, MVT RetVT) {
95 switch (VT.SimpleTy) {
96 case MVT::i32: return fastEmit_ISD_GET_FPENV_MVT_i32_(RetVT);
97 default: return Register();
98 }
99}
100
101// FastEmit functions for ISD::GET_FPMODE.
102
103Register fastEmit_ISD_GET_FPMODE_MVT_i32_(MVT RetVT) {
104 if (RetVT.SimpleTy != MVT::i32)
105 return Register();
106 return fastEmitInst_(MachineInstOpcode: ARM::VMRS, RC: &ARM::GPRnopcRegClass);
107}
108
109Register fastEmit_ISD_GET_FPMODE_(MVT VT, MVT RetVT) {
110 switch (VT.SimpleTy) {
111 case MVT::i32: return fastEmit_ISD_GET_FPMODE_MVT_i32_(RetVT);
112 default: return Register();
113 }
114}
115
116// Top-level FastEmit function.
117
118Register fastEmit_(MVT VT, MVT RetVT, unsigned Opcode) override {
119 switch (Opcode) {
120 case ISD::GET_FPENV: return fastEmit_ISD_GET_FPENV_(VT, RetVT);
121 case ISD::GET_FPMODE: return fastEmit_ISD_GET_FPMODE_(VT, RetVT);
122 default: return Register();
123 }
124}
125
126// FastEmit functions for ARMISD::CALL.
127
128Register fastEmit_ARMISD_CALL_MVT_i32_r(MVT RetVT, Register Op0) {
129 if (RetVT.SimpleTy != MVT::isVoid)
130 return Register();
131 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
132 return fastEmitInst_r(MachineInstOpcode: ARM::BLX_noip, RC: &ARM::GPRnoipRegClass, Op0);
133 }
134 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( !MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
135 return fastEmitInst_r(MachineInstOpcode: ARM::BLX, RC: &ARM::GPRRegClass, Op0);
136 }
137 return Register();
138}
139
140Register fastEmit_ARMISD_CALL_r(MVT VT, MVT RetVT, Register Op0) {
141 switch (VT.SimpleTy) {
142 case MVT::i32: return fastEmit_ARMISD_CALL_MVT_i32_r(RetVT, Op0);
143 default: return Register();
144 }
145}
146
147// FastEmit functions for ARMISD::CALL_NOLINK.
148
149Register fastEmit_ARMISD_CALL_NOLINK_MVT_i32_r(MVT RetVT, Register Op0) {
150 if (RetVT.SimpleTy != MVT::isVoid)
151 return Register();
152 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
153 return fastEmitInst_r(MachineInstOpcode: ARM::tBX_CALL, RC: &ARM::tGPRRegClass, Op0);
154 }
155 if ((!Subtarget->isThumb()) && (!Subtarget->hasV4TOps())) {
156 return fastEmitInst_r(MachineInstOpcode: ARM::BMOVPCRX_CALL, RC: &ARM::tGPRRegClass, Op0);
157 }
158 if ((Subtarget->hasV4TOps()) && (!Subtarget->isThumb())) {
159 return fastEmitInst_r(MachineInstOpcode: ARM::BX_CALL, RC: &ARM::tGPRRegClass, Op0);
160 }
161 return Register();
162}
163
164Register fastEmit_ARMISD_CALL_NOLINK_r(MVT VT, MVT RetVT, Register Op0) {
165 switch (VT.SimpleTy) {
166 case MVT::i32: return fastEmit_ARMISD_CALL_NOLINK_MVT_i32_r(RetVT, Op0);
167 default: return Register();
168 }
169}
170
171// FastEmit functions for ARMISD::CALL_PRED.
172
173Register fastEmit_ARMISD_CALL_PRED_MVT_i32_r(MVT RetVT, Register Op0) {
174 if (RetVT.SimpleTy != MVT::isVoid)
175 return Register();
176 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
177 return fastEmitInst_r(MachineInstOpcode: ARM::BLX_pred_noip, RC: &ARM::GPRnoipRegClass, Op0);
178 }
179 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( !MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
180 return fastEmitInst_r(MachineInstOpcode: ARM::BLX_pred, RC: &ARM::GPRRegClass, Op0);
181 }
182 return Register();
183}
184
185Register fastEmit_ARMISD_CALL_PRED_r(MVT VT, MVT RetVT, Register Op0) {
186 switch (VT.SimpleTy) {
187 case MVT::i32: return fastEmit_ARMISD_CALL_PRED_MVT_i32_r(RetVT, Op0);
188 default: return Register();
189 }
190}
191
192// FastEmit functions for ARMISD::CMPFPEw0.
193
194Register fastEmit_ARMISD_CMPFPEw0_MVT_f16_r(MVT RetVT, Register Op0) {
195 if (RetVT.SimpleTy != MVT::i32)
196 return Register();
197 if ((Subtarget->hasFullFP16())) {
198 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPEZH, RC: &ARM::HPRRegClass, Op0);
199 }
200 return Register();
201}
202
203Register fastEmit_ARMISD_CMPFPEw0_MVT_f32_r(MVT RetVT, Register Op0) {
204 if (RetVT.SimpleTy != MVT::i32)
205 return Register();
206 if ((Subtarget->hasVFP2Base())) {
207 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPEZS, RC: &ARM::SPRRegClass, Op0);
208 }
209 return Register();
210}
211
212Register fastEmit_ARMISD_CMPFPEw0_MVT_f64_r(MVT RetVT, Register Op0) {
213 if (RetVT.SimpleTy != MVT::i32)
214 return Register();
215 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
216 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPEZD, RC: &ARM::DPRRegClass, Op0);
217 }
218 return Register();
219}
220
221Register fastEmit_ARMISD_CMPFPEw0_r(MVT VT, MVT RetVT, Register Op0) {
222 switch (VT.SimpleTy) {
223 case MVT::f16: return fastEmit_ARMISD_CMPFPEw0_MVT_f16_r(RetVT, Op0);
224 case MVT::f32: return fastEmit_ARMISD_CMPFPEw0_MVT_f32_r(RetVT, Op0);
225 case MVT::f64: return fastEmit_ARMISD_CMPFPEw0_MVT_f64_r(RetVT, Op0);
226 default: return Register();
227 }
228}
229
230// FastEmit functions for ARMISD::CMPFPw0.
231
232Register fastEmit_ARMISD_CMPFPw0_MVT_f16_r(MVT RetVT, Register Op0) {
233 if (RetVT.SimpleTy != MVT::i32)
234 return Register();
235 if ((Subtarget->hasFullFP16())) {
236 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPZH, RC: &ARM::HPRRegClass, Op0);
237 }
238 return Register();
239}
240
241Register fastEmit_ARMISD_CMPFPw0_MVT_f32_r(MVT RetVT, Register Op0) {
242 if (RetVT.SimpleTy != MVT::i32)
243 return Register();
244 if ((Subtarget->hasVFP2Base())) {
245 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPZS, RC: &ARM::SPRRegClass, Op0);
246 }
247 return Register();
248}
249
250Register fastEmit_ARMISD_CMPFPw0_MVT_f64_r(MVT RetVT, Register Op0) {
251 if (RetVT.SimpleTy != MVT::i32)
252 return Register();
253 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
254 return fastEmitInst_r(MachineInstOpcode: ARM::VCMPZD, RC: &ARM::DPRRegClass, Op0);
255 }
256 return Register();
257}
258
259Register fastEmit_ARMISD_CMPFPw0_r(MVT VT, MVT RetVT, Register Op0) {
260 switch (VT.SimpleTy) {
261 case MVT::f16: return fastEmit_ARMISD_CMPFPw0_MVT_f16_r(RetVT, Op0);
262 case MVT::f32: return fastEmit_ARMISD_CMPFPw0_MVT_f32_r(RetVT, Op0);
263 case MVT::f64: return fastEmit_ARMISD_CMPFPw0_MVT_f64_r(RetVT, Op0);
264 default: return Register();
265 }
266}
267
268// FastEmit functions for ARMISD::VADDVs.
269
270Register fastEmit_ARMISD_VADDVs_MVT_v16i8_r(MVT RetVT, Register Op0) {
271 if (RetVT.SimpleTy != MVT::i32)
272 return Register();
273 if ((Subtarget->hasMVEIntegerOps())) {
274 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVs8no_acc, RC: &ARM::tGPREvenRegClass, Op0);
275 }
276 return Register();
277}
278
279Register fastEmit_ARMISD_VADDVs_MVT_v8i16_r(MVT RetVT, Register Op0) {
280 if (RetVT.SimpleTy != MVT::i32)
281 return Register();
282 if ((Subtarget->hasMVEIntegerOps())) {
283 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVs16no_acc, RC: &ARM::tGPREvenRegClass, Op0);
284 }
285 return Register();
286}
287
288Register fastEmit_ARMISD_VADDVs_MVT_v4i32_r(MVT RetVT, Register Op0) {
289 if (RetVT.SimpleTy != MVT::i32)
290 return Register();
291 if ((Subtarget->hasMVEIntegerOps())) {
292 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVs32no_acc, RC: &ARM::tGPREvenRegClass, Op0);
293 }
294 return Register();
295}
296
297Register fastEmit_ARMISD_VADDVs_r(MVT VT, MVT RetVT, Register Op0) {
298 switch (VT.SimpleTy) {
299 case MVT::v16i8: return fastEmit_ARMISD_VADDVs_MVT_v16i8_r(RetVT, Op0);
300 case MVT::v8i16: return fastEmit_ARMISD_VADDVs_MVT_v8i16_r(RetVT, Op0);
301 case MVT::v4i32: return fastEmit_ARMISD_VADDVs_MVT_v4i32_r(RetVT, Op0);
302 default: return Register();
303 }
304}
305
306// FastEmit functions for ARMISD::VADDVu.
307
308Register fastEmit_ARMISD_VADDVu_MVT_v16i8_r(MVT RetVT, Register Op0) {
309 if (RetVT.SimpleTy != MVT::i32)
310 return Register();
311 if ((Subtarget->hasMVEIntegerOps())) {
312 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu8no_acc, RC: &ARM::tGPREvenRegClass, Op0);
313 }
314 return Register();
315}
316
317Register fastEmit_ARMISD_VADDVu_MVT_v8i16_r(MVT RetVT, Register Op0) {
318 if (RetVT.SimpleTy != MVT::i32)
319 return Register();
320 if ((Subtarget->hasMVEIntegerOps())) {
321 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu16no_acc, RC: &ARM::tGPREvenRegClass, Op0);
322 }
323 return Register();
324}
325
326Register fastEmit_ARMISD_VADDVu_MVT_v4i32_r(MVT RetVT, Register Op0) {
327 if (RetVT.SimpleTy != MVT::i32)
328 return Register();
329 if ((Subtarget->hasMVEIntegerOps())) {
330 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu32no_acc, RC: &ARM::tGPREvenRegClass, Op0);
331 }
332 return Register();
333}
334
335Register fastEmit_ARMISD_VADDVu_r(MVT VT, MVT RetVT, Register Op0) {
336 switch (VT.SimpleTy) {
337 case MVT::v16i8: return fastEmit_ARMISD_VADDVu_MVT_v16i8_r(RetVT, Op0);
338 case MVT::v8i16: return fastEmit_ARMISD_VADDVu_MVT_v8i16_r(RetVT, Op0);
339 case MVT::v4i32: return fastEmit_ARMISD_VADDVu_MVT_v4i32_r(RetVT, Op0);
340 default: return Register();
341 }
342}
343
344// FastEmit functions for ARMISD::VDUP.
345
346Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i8_r(Register Op0) {
347 if ((Subtarget->hasNEON())) {
348 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP8d, RC: &ARM::DPRRegClass, Op0);
349 }
350 return Register();
351}
352
353Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v16i8_r(Register Op0) {
354 if ((Subtarget->hasMVEIntegerOps())) {
355 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP8, RC: &ARM::MQPRRegClass, Op0);
356 }
357 if ((Subtarget->hasNEON())) {
358 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP8q, RC: &ARM::QPRRegClass, Op0);
359 }
360 return Register();
361}
362
363Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i16_r(Register Op0) {
364 if ((Subtarget->hasNEON())) {
365 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP16d, RC: &ARM::DPRRegClass, Op0);
366 }
367 return Register();
368}
369
370Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i16_r(Register Op0) {
371 if ((Subtarget->hasMVEIntegerOps())) {
372 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP16, RC: &ARM::MQPRRegClass, Op0);
373 }
374 if ((Subtarget->hasNEON())) {
375 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP16q, RC: &ARM::QPRRegClass, Op0);
376 }
377 return Register();
378}
379
380Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v2i32_r(Register Op0) {
381 if ((!Subtarget->hasSlowVDUP32()) && (Subtarget->hasNEON())) {
382 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP32d, RC: &ARM::DPRRegClass, Op0);
383 }
384 return Register();
385}
386
387Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i32_r(Register Op0) {
388 if ((Subtarget->hasMVEIntegerOps())) {
389 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP32, RC: &ARM::MQPRRegClass, Op0);
390 }
391 if ((Subtarget->hasNEON())) {
392 return fastEmitInst_r(MachineInstOpcode: ARM::VDUP32q, RC: &ARM::QPRRegClass, Op0);
393 }
394 return Register();
395}
396
397Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8f16_r(Register Op0) {
398 if ((Subtarget->hasMVEIntegerOps())) {
399 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP16, RC: &ARM::MQPRRegClass, Op0);
400 }
401 return Register();
402}
403
404Register fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4f32_r(Register Op0) {
405 if ((Subtarget->hasMVEIntegerOps())) {
406 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VDUP32, RC: &ARM::MQPRRegClass, Op0);
407 }
408 return Register();
409}
410
411Register fastEmit_ARMISD_VDUP_MVT_i32_r(MVT RetVT, Register Op0) {
412switch (RetVT.SimpleTy) {
413 case MVT::v8i8: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i8_r(Op0);
414 case MVT::v16i8: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v16i8_r(Op0);
415 case MVT::v4i16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i16_r(Op0);
416 case MVT::v8i16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i16_r(Op0);
417 case MVT::v2i32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v2i32_r(Op0);
418 case MVT::v4i32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i32_r(Op0);
419 case MVT::v8f16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8f16_r(Op0);
420 case MVT::v4f32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4f32_r(Op0);
421 default: return Register();
422}
423}
424
425Register fastEmit_ARMISD_VDUP_r(MVT VT, MVT RetVT, Register Op0) {
426 switch (VT.SimpleTy) {
427 case MVT::i32: return fastEmit_ARMISD_VDUP_MVT_i32_r(RetVT, Op0);
428 default: return Register();
429 }
430}
431
432// FastEmit functions for ARMISD::VMOVSR.
433
434Register fastEmit_ARMISD_VMOVSR_MVT_i32_r(MVT RetVT, Register Op0) {
435 if (RetVT.SimpleTy != MVT::f32)
436 return Register();
437 if ((Subtarget->hasFPRegs()) && (Subtarget->preferVMOVSR() ||!Subtarget->useNEONForSinglePrecisionFP())) {
438 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVSR, RC: &ARM::SPRRegClass, Op0);
439 }
440 return Register();
441}
442
443Register fastEmit_ARMISD_VMOVSR_r(MVT VT, MVT RetVT, Register Op0) {
444 switch (VT.SimpleTy) {
445 case MVT::i32: return fastEmit_ARMISD_VMOVSR_MVT_i32_r(RetVT, Op0);
446 default: return Register();
447 }
448}
449
450// FastEmit functions for ARMISD::VMOVhr.
451
452Register fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_bf16_r(Register Op0) {
453 if ((Subtarget->hasFPRegs16())) {
454 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVHR, RC: &ARM::HPRRegClass, Op0);
455 }
456 return Register();
457}
458
459Register fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_f16_r(Register Op0) {
460 if ((Subtarget->hasFPRegs16())) {
461 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVHR, RC: &ARM::HPRRegClass, Op0);
462 }
463 return Register();
464}
465
466Register fastEmit_ARMISD_VMOVhr_MVT_i32_r(MVT RetVT, Register Op0) {
467switch (RetVT.SimpleTy) {
468 case MVT::bf16: return fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_bf16_r(Op0);
469 case MVT::f16: return fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_f16_r(Op0);
470 default: return Register();
471}
472}
473
474Register fastEmit_ARMISD_VMOVhr_r(MVT VT, MVT RetVT, Register Op0) {
475 switch (VT.SimpleTy) {
476 case MVT::i32: return fastEmit_ARMISD_VMOVhr_MVT_i32_r(RetVT, Op0);
477 default: return Register();
478 }
479}
480
481// FastEmit functions for ARMISD::VMOVrh.
482
483Register fastEmit_ARMISD_VMOVrh_MVT_bf16_r(MVT RetVT, Register Op0) {
484 if (RetVT.SimpleTy != MVT::i32)
485 return Register();
486 if ((Subtarget->hasFPRegs16())) {
487 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVRH, RC: &ARM::rGPRRegClass, Op0);
488 }
489 return Register();
490}
491
492Register fastEmit_ARMISD_VMOVrh_MVT_f16_r(MVT RetVT, Register Op0) {
493 if (RetVT.SimpleTy != MVT::i32)
494 return Register();
495 if ((Subtarget->hasFPRegs16())) {
496 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVRH, RC: &ARM::rGPRRegClass, Op0);
497 }
498 return Register();
499}
500
501Register fastEmit_ARMISD_VMOVrh_r(MVT VT, MVT RetVT, Register Op0) {
502 switch (VT.SimpleTy) {
503 case MVT::bf16: return fastEmit_ARMISD_VMOVrh_MVT_bf16_r(RetVT, Op0);
504 case MVT::f16: return fastEmit_ARMISD_VMOVrh_MVT_f16_r(RetVT, Op0);
505 default: return Register();
506 }
507}
508
509// FastEmit functions for ARMISD::VREV16.
510
511Register fastEmit_ARMISD_VREV16_MVT_v8i8_r(MVT RetVT, Register Op0) {
512 if (RetVT.SimpleTy != MVT::v8i8)
513 return Register();
514 if ((Subtarget->hasNEON())) {
515 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
516 }
517 return Register();
518}
519
520Register fastEmit_ARMISD_VREV16_MVT_v16i8_r(MVT RetVT, Register Op0) {
521 if (RetVT.SimpleTy != MVT::v16i8)
522 return Register();
523 if ((Subtarget->hasMVEIntegerOps())) {
524 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
525 }
526 if ((Subtarget->hasNEON())) {
527 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
528 }
529 return Register();
530}
531
532Register fastEmit_ARMISD_VREV16_r(MVT VT, MVT RetVT, Register Op0) {
533 switch (VT.SimpleTy) {
534 case MVT::v8i8: return fastEmit_ARMISD_VREV16_MVT_v8i8_r(RetVT, Op0);
535 case MVT::v16i8: return fastEmit_ARMISD_VREV16_MVT_v16i8_r(RetVT, Op0);
536 default: return Register();
537 }
538}
539
540// FastEmit functions for ARMISD::VREV32.
541
542Register fastEmit_ARMISD_VREV32_MVT_v8i8_r(MVT RetVT, Register Op0) {
543 if (RetVT.SimpleTy != MVT::v8i8)
544 return Register();
545 if ((Subtarget->hasNEON())) {
546 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
547 }
548 return Register();
549}
550
551Register fastEmit_ARMISD_VREV32_MVT_v16i8_r(MVT RetVT, Register Op0) {
552 if (RetVT.SimpleTy != MVT::v16i8)
553 return Register();
554 if ((Subtarget->hasMVEIntegerOps())) {
555 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
556 }
557 if ((Subtarget->hasNEON())) {
558 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
559 }
560 return Register();
561}
562
563Register fastEmit_ARMISD_VREV32_MVT_v4i16_r(MVT RetVT, Register Op0) {
564 if (RetVT.SimpleTy != MVT::v4i16)
565 return Register();
566 if ((Subtarget->hasNEON())) {
567 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
568 }
569 return Register();
570}
571
572Register fastEmit_ARMISD_VREV32_MVT_v8i16_r(MVT RetVT, Register Op0) {
573 if (RetVT.SimpleTy != MVT::v8i16)
574 return Register();
575 if ((Subtarget->hasMVEIntegerOps())) {
576 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
577 }
578 if ((Subtarget->hasNEON())) {
579 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
580 }
581 return Register();
582}
583
584Register fastEmit_ARMISD_VREV32_MVT_v4f16_r(MVT RetVT, Register Op0) {
585 if (RetVT.SimpleTy != MVT::v4f16)
586 return Register();
587 if ((Subtarget->hasNEON())) {
588 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
589 }
590 return Register();
591}
592
593Register fastEmit_ARMISD_VREV32_MVT_v8f16_r(MVT RetVT, Register Op0) {
594 if (RetVT.SimpleTy != MVT::v8f16)
595 return Register();
596 if ((Subtarget->hasMVEIntegerOps())) {
597 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
598 }
599 if ((Subtarget->hasNEON())) {
600 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
601 }
602 return Register();
603}
604
605Register fastEmit_ARMISD_VREV32_MVT_v4bf16_r(MVT RetVT, Register Op0) {
606 if (RetVT.SimpleTy != MVT::v4bf16)
607 return Register();
608 if ((Subtarget->hasNEON())) {
609 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
610 }
611 return Register();
612}
613
614Register fastEmit_ARMISD_VREV32_MVT_v8bf16_r(MVT RetVT, Register Op0) {
615 if (RetVT.SimpleTy != MVT::v8bf16)
616 return Register();
617 if ((Subtarget->hasNEON())) {
618 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
619 }
620 return Register();
621}
622
623Register fastEmit_ARMISD_VREV32_r(MVT VT, MVT RetVT, Register Op0) {
624 switch (VT.SimpleTy) {
625 case MVT::v8i8: return fastEmit_ARMISD_VREV32_MVT_v8i8_r(RetVT, Op0);
626 case MVT::v16i8: return fastEmit_ARMISD_VREV32_MVT_v16i8_r(RetVT, Op0);
627 case MVT::v4i16: return fastEmit_ARMISD_VREV32_MVT_v4i16_r(RetVT, Op0);
628 case MVT::v8i16: return fastEmit_ARMISD_VREV32_MVT_v8i16_r(RetVT, Op0);
629 case MVT::v4f16: return fastEmit_ARMISD_VREV32_MVT_v4f16_r(RetVT, Op0);
630 case MVT::v8f16: return fastEmit_ARMISD_VREV32_MVT_v8f16_r(RetVT, Op0);
631 case MVT::v4bf16: return fastEmit_ARMISD_VREV32_MVT_v4bf16_r(RetVT, Op0);
632 case MVT::v8bf16: return fastEmit_ARMISD_VREV32_MVT_v8bf16_r(RetVT, Op0);
633 default: return Register();
634 }
635}
636
637// FastEmit functions for ARMISD::VREV64.
638
639Register fastEmit_ARMISD_VREV64_MVT_v8i8_r(MVT RetVT, Register Op0) {
640 if (RetVT.SimpleTy != MVT::v8i8)
641 return Register();
642 if ((Subtarget->hasNEON())) {
643 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
644 }
645 return Register();
646}
647
648Register fastEmit_ARMISD_VREV64_MVT_v16i8_r(MVT RetVT, Register Op0) {
649 if (RetVT.SimpleTy != MVT::v16i8)
650 return Register();
651 if ((Subtarget->hasMVEIntegerOps())) {
652 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
653 }
654 if ((Subtarget->hasNEON())) {
655 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
656 }
657 return Register();
658}
659
660Register fastEmit_ARMISD_VREV64_MVT_v4i16_r(MVT RetVT, Register Op0) {
661 if (RetVT.SimpleTy != MVT::v4i16)
662 return Register();
663 if ((Subtarget->hasNEON())) {
664 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
665 }
666 return Register();
667}
668
669Register fastEmit_ARMISD_VREV64_MVT_v8i16_r(MVT RetVT, Register Op0) {
670 if (RetVT.SimpleTy != MVT::v8i16)
671 return Register();
672 if ((Subtarget->hasMVEIntegerOps())) {
673 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
674 }
675 if ((Subtarget->hasNEON())) {
676 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
677 }
678 return Register();
679}
680
681Register fastEmit_ARMISD_VREV64_MVT_v2i32_r(MVT RetVT, Register Op0) {
682 if (RetVT.SimpleTy != MVT::v2i32)
683 return Register();
684 if ((Subtarget->hasNEON())) {
685 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
686 }
687 return Register();
688}
689
690Register fastEmit_ARMISD_VREV64_MVT_v4i32_r(MVT RetVT, Register Op0) {
691 if (RetVT.SimpleTy != MVT::v4i32)
692 return Register();
693 if ((Subtarget->hasMVEIntegerOps())) {
694 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
695 }
696 if ((Subtarget->hasNEON())) {
697 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
698 }
699 return Register();
700}
701
702Register fastEmit_ARMISD_VREV64_MVT_v4f16_r(MVT RetVT, Register Op0) {
703 if (RetVT.SimpleTy != MVT::v4f16)
704 return Register();
705 if ((Subtarget->hasNEON())) {
706 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
707 }
708 return Register();
709}
710
711Register fastEmit_ARMISD_VREV64_MVT_v8f16_r(MVT RetVT, Register Op0) {
712 if (RetVT.SimpleTy != MVT::v8f16)
713 return Register();
714 if ((Subtarget->hasMVEIntegerOps())) {
715 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
716 }
717 if ((Subtarget->hasNEON())) {
718 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
719 }
720 return Register();
721}
722
723Register fastEmit_ARMISD_VREV64_MVT_v4bf16_r(MVT RetVT, Register Op0) {
724 if (RetVT.SimpleTy != MVT::v4bf16)
725 return Register();
726 if ((Subtarget->hasNEON())) {
727 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
728 }
729 return Register();
730}
731
732Register fastEmit_ARMISD_VREV64_MVT_v8bf16_r(MVT RetVT, Register Op0) {
733 if (RetVT.SimpleTy != MVT::v8bf16)
734 return Register();
735 if ((Subtarget->hasNEON())) {
736 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
737 }
738 return Register();
739}
740
741Register fastEmit_ARMISD_VREV64_MVT_v2f32_r(MVT RetVT, Register Op0) {
742 if (RetVT.SimpleTy != MVT::v2f32)
743 return Register();
744 if ((Subtarget->hasNEON())) {
745 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
746 }
747 return Register();
748}
749
750Register fastEmit_ARMISD_VREV64_MVT_v4f32_r(MVT RetVT, Register Op0) {
751 if (RetVT.SimpleTy != MVT::v4f32)
752 return Register();
753 if ((Subtarget->hasMVEIntegerOps())) {
754 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
755 }
756 if ((Subtarget->hasNEON())) {
757 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
758 }
759 return Register();
760}
761
762Register fastEmit_ARMISD_VREV64_r(MVT VT, MVT RetVT, Register Op0) {
763 switch (VT.SimpleTy) {
764 case MVT::v8i8: return fastEmit_ARMISD_VREV64_MVT_v8i8_r(RetVT, Op0);
765 case MVT::v16i8: return fastEmit_ARMISD_VREV64_MVT_v16i8_r(RetVT, Op0);
766 case MVT::v4i16: return fastEmit_ARMISD_VREV64_MVT_v4i16_r(RetVT, Op0);
767 case MVT::v8i16: return fastEmit_ARMISD_VREV64_MVT_v8i16_r(RetVT, Op0);
768 case MVT::v2i32: return fastEmit_ARMISD_VREV64_MVT_v2i32_r(RetVT, Op0);
769 case MVT::v4i32: return fastEmit_ARMISD_VREV64_MVT_v4i32_r(RetVT, Op0);
770 case MVT::v4f16: return fastEmit_ARMISD_VREV64_MVT_v4f16_r(RetVT, Op0);
771 case MVT::v8f16: return fastEmit_ARMISD_VREV64_MVT_v8f16_r(RetVT, Op0);
772 case MVT::v4bf16: return fastEmit_ARMISD_VREV64_MVT_v4bf16_r(RetVT, Op0);
773 case MVT::v8bf16: return fastEmit_ARMISD_VREV64_MVT_v8bf16_r(RetVT, Op0);
774 case MVT::v2f32: return fastEmit_ARMISD_VREV64_MVT_v2f32_r(RetVT, Op0);
775 case MVT::v4f32: return fastEmit_ARMISD_VREV64_MVT_v4f32_r(RetVT, Op0);
776 default: return Register();
777 }
778}
779
780// FastEmit functions for ARMISD::WIN__DBZCHK.
781
782Register fastEmit_ARMISD_WIN__DBZCHK_MVT_i32_r(MVT RetVT, Register Op0) {
783 if (RetVT.SimpleTy != MVT::isVoid)
784 return Register();
785 return fastEmitInst_r(MachineInstOpcode: ARM::WIN__DBZCHK, RC: &ARM::tGPRRegClass, Op0);
786}
787
788Register fastEmit_ARMISD_WIN__DBZCHK_r(MVT VT, MVT RetVT, Register Op0) {
789 switch (VT.SimpleTy) {
790 case MVT::i32: return fastEmit_ARMISD_WIN__DBZCHK_MVT_i32_r(RetVT, Op0);
791 default: return Register();
792 }
793}
794
795// FastEmit functions for ARMISD::WLSSETUP.
796
797Register fastEmit_ARMISD_WLSSETUP_MVT_i32_r(MVT RetVT, Register Op0) {
798 if (RetVT.SimpleTy != MVT::i32)
799 return Register();
800 if ((Subtarget->hasLOB()) && (Subtarget->hasV8_1MMainlineOps()) && (Subtarget->isThumb2())) {
801 return fastEmitInst_r(MachineInstOpcode: ARM::t2WhileLoopSetup, RC: &ARM::GPRlrRegClass, Op0);
802 }
803 return Register();
804}
805
806Register fastEmit_ARMISD_WLSSETUP_r(MVT VT, MVT RetVT, Register Op0) {
807 switch (VT.SimpleTy) {
808 case MVT::i32: return fastEmit_ARMISD_WLSSETUP_MVT_i32_r(RetVT, Op0);
809 default: return Register();
810 }
811}
812
813// FastEmit functions for ARMISD::tSECALL.
814
815Register fastEmit_ARMISD_tSECALL_MVT_i32_r(MVT RetVT, Register Op0) {
816 if (RetVT.SimpleTy != MVT::isVoid)
817 return Register();
818 if ((Subtarget->has8MSecExt()) && (Subtarget->isThumb())) {
819 return fastEmitInst_r(MachineInstOpcode: ARM::tBLXNS_CALL, RC: &ARM::GPRnopcRegClass, Op0);
820 }
821 return Register();
822}
823
824Register fastEmit_ARMISD_tSECALL_r(MVT VT, MVT RetVT, Register Op0) {
825 switch (VT.SimpleTy) {
826 case MVT::i32: return fastEmit_ARMISD_tSECALL_MVT_i32_r(RetVT, Op0);
827 default: return Register();
828 }
829}
830
831// FastEmit functions for ISD::ABS.
832
833Register fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, Register Op0) {
834 if (RetVT.SimpleTy != MVT::v8i8)
835 return Register();
836 if ((Subtarget->hasNEON())) {
837 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv8i8, RC: &ARM::DPRRegClass, Op0);
838 }
839 return Register();
840}
841
842Register fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, Register Op0) {
843 if (RetVT.SimpleTy != MVT::v16i8)
844 return Register();
845 if ((Subtarget->hasMVEIntegerOps())) {
846 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSs8, RC: &ARM::MQPRRegClass, Op0);
847 }
848 if ((Subtarget->hasNEON())) {
849 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv16i8, RC: &ARM::QPRRegClass, Op0);
850 }
851 return Register();
852}
853
854Register fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, Register Op0) {
855 if (RetVT.SimpleTy != MVT::v4i16)
856 return Register();
857 if ((Subtarget->hasNEON())) {
858 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv4i16, RC: &ARM::DPRRegClass, Op0);
859 }
860 return Register();
861}
862
863Register fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, Register Op0) {
864 if (RetVT.SimpleTy != MVT::v8i16)
865 return Register();
866 if ((Subtarget->hasMVEIntegerOps())) {
867 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSs16, RC: &ARM::MQPRRegClass, Op0);
868 }
869 if ((Subtarget->hasNEON())) {
870 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv8i16, RC: &ARM::QPRRegClass, Op0);
871 }
872 return Register();
873}
874
875Register fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, Register Op0) {
876 if (RetVT.SimpleTy != MVT::v2i32)
877 return Register();
878 if ((Subtarget->hasNEON())) {
879 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv2i32, RC: &ARM::DPRRegClass, Op0);
880 }
881 return Register();
882}
883
884Register fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, Register Op0) {
885 if (RetVT.SimpleTy != MVT::v4i32)
886 return Register();
887 if ((Subtarget->hasMVEIntegerOps())) {
888 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSs32, RC: &ARM::MQPRRegClass, Op0);
889 }
890 if ((Subtarget->hasNEON())) {
891 return fastEmitInst_r(MachineInstOpcode: ARM::VABSv4i32, RC: &ARM::QPRRegClass, Op0);
892 }
893 return Register();
894}
895
896Register fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, Register Op0) {
897 switch (VT.SimpleTy) {
898 case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0);
899 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
900 case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0);
901 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
902 case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0);
903 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
904 default: return Register();
905 }
906}
907
908// FastEmit functions for ISD::ANY_EXTEND.
909
910Register fastEmit_ISD_ANY_EXTEND_MVT_v8i8_r(MVT RetVT, Register Op0) {
911 if (RetVT.SimpleTy != MVT::v8i16)
912 return Register();
913 if ((Subtarget->hasNEON())) {
914 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv8i16, RC: &ARM::QPRRegClass, Op0);
915 }
916 return Register();
917}
918
919Register fastEmit_ISD_ANY_EXTEND_MVT_v4i16_r(MVT RetVT, Register Op0) {
920 if (RetVT.SimpleTy != MVT::v4i32)
921 return Register();
922 if ((Subtarget->hasNEON())) {
923 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv4i32, RC: &ARM::QPRRegClass, Op0);
924 }
925 return Register();
926}
927
928Register fastEmit_ISD_ANY_EXTEND_MVT_v2i32_r(MVT RetVT, Register Op0) {
929 if (RetVT.SimpleTy != MVT::v2i64)
930 return Register();
931 if ((Subtarget->hasNEON())) {
932 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv2i64, RC: &ARM::QPRRegClass, Op0);
933 }
934 return Register();
935}
936
937Register fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
938 switch (VT.SimpleTy) {
939 case MVT::v8i8: return fastEmit_ISD_ANY_EXTEND_MVT_v8i8_r(RetVT, Op0);
940 case MVT::v4i16: return fastEmit_ISD_ANY_EXTEND_MVT_v4i16_r(RetVT, Op0);
941 case MVT::v2i32: return fastEmit_ISD_ANY_EXTEND_MVT_v2i32_r(RetVT, Op0);
942 default: return Register();
943 }
944}
945
946// FastEmit functions for ISD::BITCAST.
947
948Register fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, Register Op0) {
949 if (RetVT.SimpleTy != MVT::f32)
950 return Register();
951 if ((Subtarget->hasFPRegs()) && (Subtarget->preferVMOVSR() ||!Subtarget->useNEONForSinglePrecisionFP())) {
952 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVSR, RC: &ARM::SPRRegClass, Op0);
953 }
954 return Register();
955}
956
957Register fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, Register Op0) {
958 if (RetVT.SimpleTy != MVT::i32)
959 return Register();
960 if ((Subtarget->hasFPRegs())) {
961 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVRS, RC: &ARM::GPRRegClass, Op0);
962 }
963 return Register();
964}
965
966Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Register Op0) {
967 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
968 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
969 }
970 return Register();
971}
972
973Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Register Op0) {
974 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
975 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
976 }
977 return Register();
978}
979
980Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Register Op0) {
981 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
982 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
983 }
984 return Register();
985}
986
987Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Register Op0) {
988 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
989 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
990 }
991 return Register();
992}
993
994Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Register Op0) {
995 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
996 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
997 }
998 return Register();
999}
1000
1001Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Register Op0) {
1002 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1003 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1004 }
1005 return Register();
1006}
1007
1008Register fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, Register Op0) {
1009switch (RetVT.SimpleTy) {
1010 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0);
1011 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0);
1012 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0);
1013 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0);
1014 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Op0);
1015 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0);
1016 default: return Register();
1017}
1018}
1019
1020Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Register Op0) {
1021 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1022 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
1023 }
1024 return Register();
1025}
1026
1027Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Register Op0) {
1028 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1029 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1030 }
1031 return Register();
1032}
1033
1034Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Register Op0) {
1035 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1036 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1037 }
1038 return Register();
1039}
1040
1041Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Register Op0) {
1042 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1043 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
1044 }
1045 return Register();
1046}
1047
1048Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Register Op0) {
1049 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1050 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1051 }
1052 return Register();
1053}
1054
1055Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Register Op0) {
1056 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1057 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1058 }
1059 return Register();
1060}
1061
1062Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Register Op0) {
1063 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1064 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1065 }
1066 return Register();
1067}
1068
1069Register fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, Register Op0) {
1070switch (RetVT.SimpleTy) {
1071 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0);
1072 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0);
1073 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0);
1074 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0);
1075 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0);
1076 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Op0);
1077 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0);
1078 default: return Register();
1079}
1080}
1081
1082Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Register Op0) {
1083 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1084 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1085 }
1086 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1087 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1088 }
1089 return Register();
1090}
1091
1092Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Register Op0) {
1093 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1094 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1095 }
1096 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1097 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1098 }
1099 return Register();
1100}
1101
1102Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Register Op0) {
1103 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1104 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1105 }
1106 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1107 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1108 }
1109 return Register();
1110}
1111
1112Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Register Op0) {
1113 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1114 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1115 }
1116 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1117 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1118 }
1119 return Register();
1120}
1121
1122Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Register Op0) {
1123 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1124 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1125 }
1126 return Register();
1127}
1128
1129Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Register Op0) {
1130 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1131 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1132 }
1133 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1134 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1135 }
1136 return Register();
1137}
1138
1139Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Register Op0) {
1140 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1141 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1142 }
1143 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1144 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1145 }
1146 return Register();
1147}
1148
1149Register fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, Register Op0) {
1150switch (RetVT.SimpleTy) {
1151 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0);
1152 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0);
1153 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0);
1154 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0);
1155 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Op0);
1156 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0);
1157 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0);
1158 default: return Register();
1159}
1160}
1161
1162Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Register Op0) {
1163 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1164 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1165 }
1166 return Register();
1167}
1168
1169Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Register Op0) {
1170 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1171 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1172 }
1173 return Register();
1174}
1175
1176Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Register Op0) {
1177 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1178 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1179 }
1180 return Register();
1181}
1182
1183Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Register Op0) {
1184 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1185 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1186 }
1187 return Register();
1188}
1189
1190Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Register Op0) {
1191 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1192 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1193 }
1194 return Register();
1195}
1196
1197Register fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, Register Op0) {
1198switch (RetVT.SimpleTy) {
1199 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0);
1200 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0);
1201 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0);
1202 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0);
1203 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0);
1204 default: return Register();
1205}
1206}
1207
1208Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Register Op0) {
1209 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1210 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1211 }
1212 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1213 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1214 }
1215 return Register();
1216}
1217
1218Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Register Op0) {
1219 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1220 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1221 }
1222 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1223 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1224 }
1225 return Register();
1226}
1227
1228Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Register Op0) {
1229 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1230 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1231 }
1232 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1233 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1234 }
1235 return Register();
1236}
1237
1238Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Register Op0) {
1239 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1240 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1241 }
1242 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1243 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1244 }
1245 return Register();
1246}
1247
1248Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Register Op0) {
1249 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1250 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1251 }
1252 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1253 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1254 }
1255 return Register();
1256}
1257
1258Register fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, Register Op0) {
1259switch (RetVT.SimpleTy) {
1260 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0);
1261 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0);
1262 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0);
1263 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0);
1264 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0);
1265 default: return Register();
1266}
1267}
1268
1269Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Register Op0) {
1270 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1271 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1272 }
1273 return Register();
1274}
1275
1276Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Register Op0) {
1277 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1278 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1279 }
1280 return Register();
1281}
1282
1283Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Register Op0) {
1284 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1285 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1286 }
1287 return Register();
1288}
1289
1290Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Register Op0) {
1291 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1292 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1293 }
1294 return Register();
1295}
1296
1297Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Register Op0) {
1298 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1299 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1300 }
1301 return Register();
1302}
1303
1304Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Register Op0) {
1305 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1306 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1307 }
1308 return Register();
1309}
1310
1311Register fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, Register Op0) {
1312switch (RetVT.SimpleTy) {
1313 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0);
1314 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0);
1315 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0);
1316 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0);
1317 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0);
1318 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Op0);
1319 default: return Register();
1320}
1321}
1322
1323Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Register Op0) {
1324 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1325 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1326 }
1327 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1328 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1329 }
1330 return Register();
1331}
1332
1333Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Register Op0) {
1334 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1335 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1336 }
1337 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1338 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1339 }
1340 return Register();
1341}
1342
1343Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Register Op0) {
1344 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1345 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1346 }
1347 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1348 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1349 }
1350 return Register();
1351}
1352
1353Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Register Op0) {
1354 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1355 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1356 }
1357 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1358 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1359 }
1360 return Register();
1361}
1362
1363Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Register Op0) {
1364 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1365 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1366 }
1367 return Register();
1368}
1369
1370Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Register Op0) {
1371 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1372 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1373 }
1374 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1375 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1376 }
1377 return Register();
1378}
1379
1380Register fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, Register Op0) {
1381switch (RetVT.SimpleTy) {
1382 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0);
1383 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0);
1384 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0);
1385 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0);
1386 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Op0);
1387 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0);
1388 default: return Register();
1389}
1390}
1391
1392Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Register Op0) {
1393 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1394 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d8, RC: &ARM::DPRRegClass, Op0);
1395 }
1396 return Register();
1397}
1398
1399Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Register Op0) {
1400 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1401 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1402 }
1403 return Register();
1404}
1405
1406Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Register Op0) {
1407 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1408 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1409 }
1410 return Register();
1411}
1412
1413Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Register Op0) {
1414 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1415 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1416 }
1417 return Register();
1418}
1419
1420Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Register Op0) {
1421 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1422 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1423 }
1424 return Register();
1425}
1426
1427Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Register Op0) {
1428 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1429 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1430 }
1431 return Register();
1432}
1433
1434Register fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, Register Op0) {
1435switch (RetVT.SimpleTy) {
1436 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0);
1437 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0);
1438 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0);
1439 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0);
1440 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Op0);
1441 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0);
1442 default: return Register();
1443}
1444}
1445
1446Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Register Op0) {
1447 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1448 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1449 }
1450 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1451 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1452 }
1453 return Register();
1454}
1455
1456Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Register Op0) {
1457 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1458 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1459 }
1460 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1461 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1462 }
1463 return Register();
1464}
1465
1466Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Register Op0) {
1467 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1468 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1469 }
1470 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1471 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1472 }
1473 return Register();
1474}
1475
1476Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Register Op0) {
1477 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1478 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1479 }
1480 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1481 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1482 }
1483 return Register();
1484}
1485
1486Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Register Op0) {
1487 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1488 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1489 }
1490 return Register();
1491}
1492
1493Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Register Op0) {
1494 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1495 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1496 }
1497 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1498 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1499 }
1500 return Register();
1501}
1502
1503Register fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, Register Op0) {
1504switch (RetVT.SimpleTy) {
1505 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0);
1506 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0);
1507 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0);
1508 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0);
1509 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Op0);
1510 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0);
1511 default: return Register();
1512}
1513}
1514
1515Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Register Op0) {
1516 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1517 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1518 }
1519 return Register();
1520}
1521
1522Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Register Op0) {
1523 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1524 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1525 }
1526 return Register();
1527}
1528
1529Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Register Op0) {
1530 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1531 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1532 }
1533 return Register();
1534}
1535
1536Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Register Op0) {
1537 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1538 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1539 }
1540 return Register();
1541}
1542
1543Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Register Op0) {
1544 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1545 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1546 }
1547 return Register();
1548}
1549
1550Register fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, Register Op0) {
1551switch (RetVT.SimpleTy) {
1552 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0);
1553 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0);
1554 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0);
1555 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0);
1556 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0);
1557 default: return Register();
1558}
1559}
1560
1561Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Register Op0) {
1562 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1563 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1564 }
1565 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1566 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1567 }
1568 return Register();
1569}
1570
1571Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Register Op0) {
1572 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1573 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1574 }
1575 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1576 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1577 }
1578 return Register();
1579}
1580
1581Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Register Op0) {
1582 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1583 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1584 }
1585 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1586 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1587 }
1588 return Register();
1589}
1590
1591Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Register Op0) {
1592 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1593 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1594 }
1595 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1596 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1597 }
1598 return Register();
1599}
1600
1601Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Register Op0) {
1602 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1603 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1604 }
1605 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1606 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1607 }
1608 return Register();
1609}
1610
1611Register fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, Register Op0) {
1612switch (RetVT.SimpleTy) {
1613 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0);
1614 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0);
1615 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0);
1616 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0);
1617 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0);
1618 default: return Register();
1619}
1620}
1621
1622Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Register Op0) {
1623 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1624 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1625 }
1626 return Register();
1627}
1628
1629Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Register Op0) {
1630 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1631 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16d8, RC: &ARM::DPRRegClass, Op0);
1632 }
1633 return Register();
1634}
1635
1636Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Register Op0) {
1637 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1638 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1639 }
1640 return Register();
1641}
1642
1643Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Register Op0) {
1644 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1645 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d16, RC: &ARM::DPRRegClass, Op0);
1646 }
1647 return Register();
1648}
1649
1650Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Register Op0) {
1651 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1652 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1653 }
1654 return Register();
1655}
1656
1657Register fastEmit_ISD_BITCAST_MVT_v4bf16_r(MVT RetVT, Register Op0) {
1658switch (RetVT.SimpleTy) {
1659 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Op0);
1660 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Op0);
1661 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Op0);
1662 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Op0);
1663 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Op0);
1664 default: return Register();
1665}
1666}
1667
1668Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Register Op0) {
1669 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1670 return fastEmitInst_r(MachineInstOpcode: ARM::VREV16q8, RC: &ARM::QPRRegClass, Op0);
1671 }
1672 return Register();
1673}
1674
1675Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Register Op0) {
1676 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1677 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1678 }
1679 return Register();
1680}
1681
1682Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Register Op0) {
1683 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1684 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1685 }
1686 return Register();
1687}
1688
1689Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Register Op0) {
1690 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1691 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1692 }
1693 return Register();
1694}
1695
1696Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Register Op0) {
1697 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1698 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1699 }
1700 return Register();
1701}
1702
1703Register fastEmit_ISD_BITCAST_MVT_v8bf16_r(MVT RetVT, Register Op0) {
1704switch (RetVT.SimpleTy) {
1705 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Op0);
1706 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Op0);
1707 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Op0);
1708 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Op0);
1709 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Op0);
1710 default: return Register();
1711}
1712}
1713
1714Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Register Op0) {
1715 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1716 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1717 }
1718 return Register();
1719}
1720
1721Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Register Op0) {
1722 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1723 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d8, RC: &ARM::DPRRegClass, Op0);
1724 }
1725 return Register();
1726}
1727
1728Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Register Op0) {
1729 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1730 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1731 }
1732 return Register();
1733}
1734
1735Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Register Op0) {
1736 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1737 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64d32, RC: &ARM::DPRRegClass, Op0);
1738 }
1739 return Register();
1740}
1741
1742Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Register Op0) {
1743 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1744 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1745 }
1746 return Register();
1747}
1748
1749Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Register Op0) {
1750 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1751 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32d16, RC: &ARM::DPRRegClass, Op0);
1752 }
1753 return Register();
1754}
1755
1756Register fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, Register Op0) {
1757switch (RetVT.SimpleTy) {
1758 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0);
1759 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0);
1760 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0);
1761 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0);
1762 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0);
1763 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Op0);
1764 default: return Register();
1765}
1766}
1767
1768Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Register Op0) {
1769 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1770 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
1771 }
1772 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1773 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q8, RC: &ARM::QPRRegClass, Op0);
1774 }
1775 return Register();
1776}
1777
1778Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Register Op0) {
1779 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1780 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1781 }
1782 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1783 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1784 }
1785 return Register();
1786}
1787
1788Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Register Op0) {
1789 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1790 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1791 }
1792 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1793 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1794 }
1795 return Register();
1796}
1797
1798Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Register Op0) {
1799 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1800 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_16, RC: &ARM::MQPRRegClass, Op0);
1801 }
1802 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1803 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1804 }
1805 return Register();
1806}
1807
1808Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Register Op0) {
1809 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1810 return fastEmitInst_r(MachineInstOpcode: ARM::VREV32q16, RC: &ARM::QPRRegClass, Op0);
1811 }
1812 return Register();
1813}
1814
1815Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Register Op0) {
1816 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1817 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1818 }
1819 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1820 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1821 }
1822 return Register();
1823}
1824
1825Register fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, Register Op0) {
1826switch (RetVT.SimpleTy) {
1827 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0);
1828 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0);
1829 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0);
1830 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0);
1831 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Op0);
1832 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0);
1833 default: return Register();
1834}
1835}
1836
1837Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Register Op0) {
1838 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1839 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_8, RC: &ARM::MQPRRegClass, Op0);
1840 }
1841 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1842 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q8, RC: &ARM::QPRRegClass, Op0);
1843 }
1844 return Register();
1845}
1846
1847Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Register Op0) {
1848 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1849 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1850 }
1851 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1852 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1853 }
1854 return Register();
1855}
1856
1857Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Register Op0) {
1858 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1859 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1860 }
1861 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1862 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1863 }
1864 return Register();
1865}
1866
1867Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Register Op0) {
1868 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1869 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_16, RC: &ARM::MQPRRegClass, Op0);
1870 }
1871 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1872 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1873 }
1874 return Register();
1875}
1876
1877Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Register Op0) {
1878 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1879 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q16, RC: &ARM::QPRRegClass, Op0);
1880 }
1881 return Register();
1882}
1883
1884Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Register Op0) {
1885 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1886 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV64_32, RC: &ARM::MQPRRegClass, Op0);
1887 }
1888 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1889 return fastEmitInst_r(MachineInstOpcode: ARM::VREV64q32, RC: &ARM::QPRRegClass, Op0);
1890 }
1891 return Register();
1892}
1893
1894Register fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, Register Op0) {
1895switch (RetVT.SimpleTy) {
1896 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0);
1897 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0);
1898 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0);
1899 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0);
1900 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Op0);
1901 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0);
1902 default: return Register();
1903}
1904}
1905
1906Register fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, Register Op0) {
1907 switch (VT.SimpleTy) {
1908 case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0);
1909 case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0);
1910 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
1911 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0);
1912 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0);
1913 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0);
1914 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0);
1915 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0);
1916 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0);
1917 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0);
1918 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0);
1919 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0);
1920 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0);
1921 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v4bf16_r(RetVT, Op0);
1922 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v8bf16_r(RetVT, Op0);
1923 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0);
1924 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0);
1925 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0);
1926 default: return Register();
1927 }
1928}
1929
1930// FastEmit functions for ISD::BITREVERSE.
1931
1932Register fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, Register Op0) {
1933 if (RetVT.SimpleTy != MVT::i32)
1934 return Register();
1935 if ((Subtarget->isThumb2())) {
1936 return fastEmitInst_r(MachineInstOpcode: ARM::t2RBIT, RC: &ARM::rGPRRegClass, Op0);
1937 }
1938 if ((Subtarget->hasV6T2Ops()) && (!Subtarget->isThumb())) {
1939 return fastEmitInst_r(MachineInstOpcode: ARM::RBIT, RC: &ARM::GPRRegClass, Op0);
1940 }
1941 return Register();
1942}
1943
1944Register fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, Register Op0) {
1945 switch (VT.SimpleTy) {
1946 case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0);
1947 default: return Register();
1948 }
1949}
1950
1951// FastEmit functions for ISD::BRIND.
1952
1953Register fastEmit_ISD_BRIND_MVT_i32_r(MVT RetVT, Register Op0) {
1954 if (RetVT.SimpleTy != MVT::isVoid)
1955 return Register();
1956 if ((Subtarget->isThumb())) {
1957 return fastEmitInst_r(MachineInstOpcode: ARM::tBRIND, RC: &ARM::GPRRegClass, Op0);
1958 }
1959 if ((!Subtarget->isThumb()) && (!Subtarget->hasV4TOps())) {
1960 return fastEmitInst_r(MachineInstOpcode: ARM::MOVPCRX, RC: &ARM::GPRRegClass, Op0);
1961 }
1962 if ((Subtarget->hasV4TOps()) && (!Subtarget->isThumb())) {
1963 return fastEmitInst_r(MachineInstOpcode: ARM::BX, RC: &ARM::GPRRegClass, Op0);
1964 }
1965 return Register();
1966}
1967
1968Register fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, Register Op0) {
1969 switch (VT.SimpleTy) {
1970 case MVT::i32: return fastEmit_ISD_BRIND_MVT_i32_r(RetVT, Op0);
1971 default: return Register();
1972 }
1973}
1974
1975// FastEmit functions for ISD::BSWAP.
1976
1977Register fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, Register Op0) {
1978 if (RetVT.SimpleTy != MVT::i32)
1979 return Register();
1980 if ((Subtarget->isThumb2())) {
1981 return fastEmitInst_r(MachineInstOpcode: ARM::t2REV, RC: &ARM::rGPRRegClass, Op0);
1982 }
1983 if ((Subtarget->hasV6Ops()) && (Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
1984 return fastEmitInst_r(MachineInstOpcode: ARM::tREV, RC: &ARM::tGPRRegClass, Op0);
1985 }
1986 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
1987 return fastEmitInst_r(MachineInstOpcode: ARM::REV, RC: &ARM::GPRRegClass, Op0);
1988 }
1989 return Register();
1990}
1991
1992Register fastEmit_ISD_BSWAP_MVT_v8i16_r(MVT RetVT, Register Op0) {
1993 if (RetVT.SimpleTy != MVT::v8i16)
1994 return Register();
1995 if ((Subtarget->hasMVEIntegerOps())) {
1996 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV16_8, RC: &ARM::MQPRRegClass, Op0);
1997 }
1998 return Register();
1999}
2000
2001Register fastEmit_ISD_BSWAP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2002 if (RetVT.SimpleTy != MVT::v4i32)
2003 return Register();
2004 if ((Subtarget->hasMVEIntegerOps())) {
2005 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VREV32_8, RC: &ARM::MQPRRegClass, Op0);
2006 }
2007 return Register();
2008}
2009
2010Register fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, Register Op0) {
2011 switch (VT.SimpleTy) {
2012 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
2013 case MVT::v8i16: return fastEmit_ISD_BSWAP_MVT_v8i16_r(RetVT, Op0);
2014 case MVT::v4i32: return fastEmit_ISD_BSWAP_MVT_v4i32_r(RetVT, Op0);
2015 default: return Register();
2016 }
2017}
2018
2019// FastEmit functions for ISD::CTLS.
2020
2021Register fastEmit_ISD_CTLS_MVT_v8i8_r(MVT RetVT, Register Op0) {
2022 if (RetVT.SimpleTy != MVT::v8i8)
2023 return Register();
2024 if ((Subtarget->hasNEON())) {
2025 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv8i8, RC: &ARM::DPRRegClass, Op0);
2026 }
2027 return Register();
2028}
2029
2030Register fastEmit_ISD_CTLS_MVT_v16i8_r(MVT RetVT, Register Op0) {
2031 if (RetVT.SimpleTy != MVT::v16i8)
2032 return Register();
2033 if ((Subtarget->hasMVEIntegerOps())) {
2034 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLSs8, RC: &ARM::MQPRRegClass, Op0);
2035 }
2036 if ((Subtarget->hasNEON())) {
2037 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv16i8, RC: &ARM::QPRRegClass, Op0);
2038 }
2039 return Register();
2040}
2041
2042Register fastEmit_ISD_CTLS_MVT_v4i16_r(MVT RetVT, Register Op0) {
2043 if (RetVT.SimpleTy != MVT::v4i16)
2044 return Register();
2045 if ((Subtarget->hasNEON())) {
2046 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv4i16, RC: &ARM::DPRRegClass, Op0);
2047 }
2048 return Register();
2049}
2050
2051Register fastEmit_ISD_CTLS_MVT_v8i16_r(MVT RetVT, Register Op0) {
2052 if (RetVT.SimpleTy != MVT::v8i16)
2053 return Register();
2054 if ((Subtarget->hasMVEIntegerOps())) {
2055 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLSs16, RC: &ARM::MQPRRegClass, Op0);
2056 }
2057 if ((Subtarget->hasNEON())) {
2058 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv8i16, RC: &ARM::QPRRegClass, Op0);
2059 }
2060 return Register();
2061}
2062
2063Register fastEmit_ISD_CTLS_MVT_v2i32_r(MVT RetVT, Register Op0) {
2064 if (RetVT.SimpleTy != MVT::v2i32)
2065 return Register();
2066 if ((Subtarget->hasNEON())) {
2067 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv2i32, RC: &ARM::DPRRegClass, Op0);
2068 }
2069 return Register();
2070}
2071
2072Register fastEmit_ISD_CTLS_MVT_v4i32_r(MVT RetVT, Register Op0) {
2073 if (RetVT.SimpleTy != MVT::v4i32)
2074 return Register();
2075 if ((Subtarget->hasMVEIntegerOps())) {
2076 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLSs32, RC: &ARM::MQPRRegClass, Op0);
2077 }
2078 if ((Subtarget->hasNEON())) {
2079 return fastEmitInst_r(MachineInstOpcode: ARM::VCLSv4i32, RC: &ARM::QPRRegClass, Op0);
2080 }
2081 return Register();
2082}
2083
2084Register fastEmit_ISD_CTLS_r(MVT VT, MVT RetVT, Register Op0) {
2085 switch (VT.SimpleTy) {
2086 case MVT::v8i8: return fastEmit_ISD_CTLS_MVT_v8i8_r(RetVT, Op0);
2087 case MVT::v16i8: return fastEmit_ISD_CTLS_MVT_v16i8_r(RetVT, Op0);
2088 case MVT::v4i16: return fastEmit_ISD_CTLS_MVT_v4i16_r(RetVT, Op0);
2089 case MVT::v8i16: return fastEmit_ISD_CTLS_MVT_v8i16_r(RetVT, Op0);
2090 case MVT::v2i32: return fastEmit_ISD_CTLS_MVT_v2i32_r(RetVT, Op0);
2091 case MVT::v4i32: return fastEmit_ISD_CTLS_MVT_v4i32_r(RetVT, Op0);
2092 default: return Register();
2093 }
2094}
2095
2096// FastEmit functions for ISD::CTLZ.
2097
2098Register fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, Register Op0) {
2099 if (RetVT.SimpleTy != MVT::i32)
2100 return Register();
2101 if ((Subtarget->isThumb2())) {
2102 return fastEmitInst_r(MachineInstOpcode: ARM::t2CLZ, RC: &ARM::rGPRRegClass, Op0);
2103 }
2104 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb())) {
2105 return fastEmitInst_r(MachineInstOpcode: ARM::CLZ, RC: &ARM::GPRRegClass, Op0);
2106 }
2107 return Register();
2108}
2109
2110Register fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, Register Op0) {
2111 if (RetVT.SimpleTy != MVT::v8i8)
2112 return Register();
2113 if ((Subtarget->hasNEON())) {
2114 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv8i8, RC: &ARM::DPRRegClass, Op0);
2115 }
2116 return Register();
2117}
2118
2119Register fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, Register Op0) {
2120 if (RetVT.SimpleTy != MVT::v16i8)
2121 return Register();
2122 if ((Subtarget->hasMVEIntegerOps())) {
2123 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLZs8, RC: &ARM::MQPRRegClass, Op0);
2124 }
2125 if ((Subtarget->hasNEON())) {
2126 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv16i8, RC: &ARM::QPRRegClass, Op0);
2127 }
2128 return Register();
2129}
2130
2131Register fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, Register Op0) {
2132 if (RetVT.SimpleTy != MVT::v4i16)
2133 return Register();
2134 if ((Subtarget->hasNEON())) {
2135 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv4i16, RC: &ARM::DPRRegClass, Op0);
2136 }
2137 return Register();
2138}
2139
2140Register fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, Register Op0) {
2141 if (RetVT.SimpleTy != MVT::v8i16)
2142 return Register();
2143 if ((Subtarget->hasMVEIntegerOps())) {
2144 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLZs16, RC: &ARM::MQPRRegClass, Op0);
2145 }
2146 if ((Subtarget->hasNEON())) {
2147 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv8i16, RC: &ARM::QPRRegClass, Op0);
2148 }
2149 return Register();
2150}
2151
2152Register fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, Register Op0) {
2153 if (RetVT.SimpleTy != MVT::v2i32)
2154 return Register();
2155 if ((Subtarget->hasNEON())) {
2156 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv2i32, RC: &ARM::DPRRegClass, Op0);
2157 }
2158 return Register();
2159}
2160
2161Register fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, Register Op0) {
2162 if (RetVT.SimpleTy != MVT::v4i32)
2163 return Register();
2164 if ((Subtarget->hasMVEIntegerOps())) {
2165 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCLZs32, RC: &ARM::MQPRRegClass, Op0);
2166 }
2167 if ((Subtarget->hasNEON())) {
2168 return fastEmitInst_r(MachineInstOpcode: ARM::VCLZv4i32, RC: &ARM::QPRRegClass, Op0);
2169 }
2170 return Register();
2171}
2172
2173Register fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, Register Op0) {
2174 switch (VT.SimpleTy) {
2175 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
2176 case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0);
2177 case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0);
2178 case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0);
2179 case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0);
2180 case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0);
2181 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
2182 default: return Register();
2183 }
2184}
2185
2186// FastEmit functions for ISD::CTPOP.
2187
2188Register fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, Register Op0) {
2189 if (RetVT.SimpleTy != MVT::v8i8)
2190 return Register();
2191 if ((Subtarget->hasNEON())) {
2192 return fastEmitInst_r(MachineInstOpcode: ARM::VCNTd, RC: &ARM::DPRRegClass, Op0);
2193 }
2194 return Register();
2195}
2196
2197Register fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, Register Op0) {
2198 if (RetVT.SimpleTy != MVT::v16i8)
2199 return Register();
2200 if ((Subtarget->hasNEON())) {
2201 return fastEmitInst_r(MachineInstOpcode: ARM::VCNTq, RC: &ARM::QPRRegClass, Op0);
2202 }
2203 return Register();
2204}
2205
2206Register fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, Register Op0) {
2207 switch (VT.SimpleTy) {
2208 case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0);
2209 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
2210 default: return Register();
2211 }
2212}
2213
2214// FastEmit functions for ISD::FABS.
2215
2216Register fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, Register Op0) {
2217 if (RetVT.SimpleTy != MVT::f16)
2218 return Register();
2219 if ((Subtarget->hasFullFP16())) {
2220 return fastEmitInst_r(MachineInstOpcode: ARM::VABSH, RC: &ARM::HPRRegClass, Op0);
2221 }
2222 return Register();
2223}
2224
2225Register fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, Register Op0) {
2226 if (RetVT.SimpleTy != MVT::f32)
2227 return Register();
2228 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
2229 return fastEmitInst_r(MachineInstOpcode: ARM::VABSS, RC: &ARM::SPRRegClass, Op0);
2230 }
2231 return Register();
2232}
2233
2234Register fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, Register Op0) {
2235 if (RetVT.SimpleTy != MVT::f64)
2236 return Register();
2237 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2238 return fastEmitInst_r(MachineInstOpcode: ARM::VABSD, RC: &ARM::DPRRegClass, Op0);
2239 }
2240 return Register();
2241}
2242
2243Register fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, Register Op0) {
2244 if (RetVT.SimpleTy != MVT::v4f16)
2245 return Register();
2246 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2247 return fastEmitInst_r(MachineInstOpcode: ARM::VABShd, RC: &ARM::DPRRegClass, Op0);
2248 }
2249 return Register();
2250}
2251
2252Register fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, Register Op0) {
2253 if (RetVT.SimpleTy != MVT::v8f16)
2254 return Register();
2255 if ((Subtarget->hasMVEIntegerOps())) {
2256 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSf16, RC: &ARM::MQPRRegClass, Op0);
2257 }
2258 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2259 return fastEmitInst_r(MachineInstOpcode: ARM::VABShq, RC: &ARM::QPRRegClass, Op0);
2260 }
2261 return Register();
2262}
2263
2264Register fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, Register Op0) {
2265 if (RetVT.SimpleTy != MVT::v2f32)
2266 return Register();
2267 if ((Subtarget->hasNEON())) {
2268 return fastEmitInst_r(MachineInstOpcode: ARM::VABSfd, RC: &ARM::DPRRegClass, Op0);
2269 }
2270 return Register();
2271}
2272
2273Register fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, Register Op0) {
2274 if (RetVT.SimpleTy != MVT::v4f32)
2275 return Register();
2276 if ((Subtarget->hasMVEIntegerOps())) {
2277 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VABSf32, RC: &ARM::MQPRRegClass, Op0);
2278 }
2279 if ((Subtarget->hasNEON())) {
2280 return fastEmitInst_r(MachineInstOpcode: ARM::VABSfq, RC: &ARM::QPRRegClass, Op0);
2281 }
2282 return Register();
2283}
2284
2285Register fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, Register Op0) {
2286 switch (VT.SimpleTy) {
2287 case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0);
2288 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
2289 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
2290 case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0);
2291 case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0);
2292 case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0);
2293 case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0);
2294 default: return Register();
2295 }
2296}
2297
2298// FastEmit functions for ISD::FCEIL.
2299
2300Register fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
2301 if (RetVT.SimpleTy != MVT::f16)
2302 return Register();
2303 if ((Subtarget->hasFullFP16())) {
2304 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPH, RC: &ARM::HPRRegClass, Op0);
2305 }
2306 return Register();
2307}
2308
2309Register fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
2310 if (RetVT.SimpleTy != MVT::f32)
2311 return Register();
2312 if ((Subtarget->hasFPARMv8Base())) {
2313 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPS, RC: &ARM::SPRRegClass, Op0);
2314 }
2315 return Register();
2316}
2317
2318Register fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
2319 if (RetVT.SimpleTy != MVT::f64)
2320 return Register();
2321 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2322 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPD, RC: &ARM::DPRRegClass, Op0);
2323 }
2324 return Register();
2325}
2326
2327Register fastEmit_ISD_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
2328 if (RetVT.SimpleTy != MVT::v4f16)
2329 return Register();
2330 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2331 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNDh, RC: &ARM::DPRRegClass, Op0);
2332 }
2333 return Register();
2334}
2335
2336Register fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
2337 if (RetVT.SimpleTy != MVT::v8f16)
2338 return Register();
2339 if ((Subtarget->hasMVEFloatOps())) {
2340 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16P, RC: &ARM::MQPRRegClass, Op0);
2341 }
2342 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2343 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNQh, RC: &ARM::QPRRegClass, Op0);
2344 }
2345 return Register();
2346}
2347
2348Register fastEmit_ISD_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
2349 if (RetVT.SimpleTy != MVT::v2f32)
2350 return Register();
2351 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2352 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNDf, RC: &ARM::DPRRegClass, Op0);
2353 }
2354 return Register();
2355}
2356
2357Register fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
2358 if (RetVT.SimpleTy != MVT::v4f32)
2359 return Register();
2360 if ((Subtarget->hasMVEFloatOps())) {
2361 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32P, RC: &ARM::MQPRRegClass, Op0);
2362 }
2363 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2364 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNQf, RC: &ARM::QPRRegClass, Op0);
2365 }
2366 return Register();
2367}
2368
2369Register fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
2370 switch (VT.SimpleTy) {
2371 case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0);
2372 case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0);
2373 case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0);
2374 case MVT::v4f16: return fastEmit_ISD_FCEIL_MVT_v4f16_r(RetVT, Op0);
2375 case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0);
2376 case MVT::v2f32: return fastEmit_ISD_FCEIL_MVT_v2f32_r(RetVT, Op0);
2377 case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0);
2378 default: return Register();
2379 }
2380}
2381
2382// FastEmit functions for ISD::FFLOOR.
2383
2384Register fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
2385 if (RetVT.SimpleTy != MVT::f16)
2386 return Register();
2387 if ((Subtarget->hasFullFP16())) {
2388 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMH, RC: &ARM::HPRRegClass, Op0);
2389 }
2390 return Register();
2391}
2392
2393Register fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
2394 if (RetVT.SimpleTy != MVT::f32)
2395 return Register();
2396 if ((Subtarget->hasFPARMv8Base())) {
2397 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMS, RC: &ARM::SPRRegClass, Op0);
2398 }
2399 return Register();
2400}
2401
2402Register fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
2403 if (RetVT.SimpleTy != MVT::f64)
2404 return Register();
2405 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2406 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMD, RC: &ARM::DPRRegClass, Op0);
2407 }
2408 return Register();
2409}
2410
2411Register fastEmit_ISD_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
2412 if (RetVT.SimpleTy != MVT::v4f16)
2413 return Register();
2414 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2415 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNDh, RC: &ARM::DPRRegClass, Op0);
2416 }
2417 return Register();
2418}
2419
2420Register fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
2421 if (RetVT.SimpleTy != MVT::v8f16)
2422 return Register();
2423 if ((Subtarget->hasMVEFloatOps())) {
2424 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16M, RC: &ARM::MQPRRegClass, Op0);
2425 }
2426 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2427 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNQh, RC: &ARM::QPRRegClass, Op0);
2428 }
2429 return Register();
2430}
2431
2432Register fastEmit_ISD_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
2433 if (RetVT.SimpleTy != MVT::v2f32)
2434 return Register();
2435 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2436 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNDf, RC: &ARM::DPRRegClass, Op0);
2437 }
2438 return Register();
2439}
2440
2441Register fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
2442 if (RetVT.SimpleTy != MVT::v4f32)
2443 return Register();
2444 if ((Subtarget->hasMVEFloatOps())) {
2445 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32M, RC: &ARM::MQPRRegClass, Op0);
2446 }
2447 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2448 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNQf, RC: &ARM::QPRRegClass, Op0);
2449 }
2450 return Register();
2451}
2452
2453Register fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
2454 switch (VT.SimpleTy) {
2455 case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0);
2456 case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0);
2457 case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0);
2458 case MVT::v4f16: return fastEmit_ISD_FFLOOR_MVT_v4f16_r(RetVT, Op0);
2459 case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0);
2460 case MVT::v2f32: return fastEmit_ISD_FFLOOR_MVT_v2f32_r(RetVT, Op0);
2461 case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0);
2462 default: return Register();
2463 }
2464}
2465
2466// FastEmit functions for ISD::FNEARBYINT.
2467
2468Register fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
2469 if (RetVT.SimpleTy != MVT::f16)
2470 return Register();
2471 if ((Subtarget->hasFullFP16())) {
2472 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRH, RC: &ARM::HPRRegClass, Op0);
2473 }
2474 return Register();
2475}
2476
2477Register fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
2478 if (RetVT.SimpleTy != MVT::f32)
2479 return Register();
2480 if ((Subtarget->hasFPARMv8Base())) {
2481 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRS, RC: &ARM::SPRRegClass, Op0);
2482 }
2483 return Register();
2484}
2485
2486Register fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
2487 if (RetVT.SimpleTy != MVT::f64)
2488 return Register();
2489 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2490 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRD, RC: &ARM::DPRRegClass, Op0);
2491 }
2492 return Register();
2493}
2494
2495Register fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
2496 switch (VT.SimpleTy) {
2497 case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0);
2498 case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0);
2499 case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0);
2500 default: return Register();
2501 }
2502}
2503
2504// FastEmit functions for ISD::FNEG.
2505
2506Register fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, Register Op0) {
2507 if (RetVT.SimpleTy != MVT::f16)
2508 return Register();
2509 if ((Subtarget->hasFullFP16())) {
2510 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGH, RC: &ARM::HPRRegClass, Op0);
2511 }
2512 return Register();
2513}
2514
2515Register fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, Register Op0) {
2516 if (RetVT.SimpleTy != MVT::f32)
2517 return Register();
2518 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
2519 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGS, RC: &ARM::SPRRegClass, Op0);
2520 }
2521 return Register();
2522}
2523
2524Register fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, Register Op0) {
2525 if (RetVT.SimpleTy != MVT::f64)
2526 return Register();
2527 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2528 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGD, RC: &ARM::DPRRegClass, Op0);
2529 }
2530 return Register();
2531}
2532
2533Register fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, Register Op0) {
2534 if (RetVT.SimpleTy != MVT::v4f16)
2535 return Register();
2536 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2537 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGhd, RC: &ARM::DPRRegClass, Op0);
2538 }
2539 return Register();
2540}
2541
2542Register fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, Register Op0) {
2543 if (RetVT.SimpleTy != MVT::v8f16)
2544 return Register();
2545 if ((Subtarget->hasMVEIntegerOps())) {
2546 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VNEGf16, RC: &ARM::MQPRRegClass, Op0);
2547 }
2548 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2549 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGhq, RC: &ARM::QPRRegClass, Op0);
2550 }
2551 return Register();
2552}
2553
2554Register fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, Register Op0) {
2555 if (RetVT.SimpleTy != MVT::v2f32)
2556 return Register();
2557 if ((Subtarget->hasNEON())) {
2558 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGfd, RC: &ARM::DPRRegClass, Op0);
2559 }
2560 return Register();
2561}
2562
2563Register fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, Register Op0) {
2564 if (RetVT.SimpleTy != MVT::v4f32)
2565 return Register();
2566 if ((Subtarget->hasMVEIntegerOps())) {
2567 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VNEGf32, RC: &ARM::MQPRRegClass, Op0);
2568 }
2569 if ((Subtarget->hasNEON())) {
2570 return fastEmitInst_r(MachineInstOpcode: ARM::VNEGf32q, RC: &ARM::QPRRegClass, Op0);
2571 }
2572 return Register();
2573}
2574
2575Register fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, Register Op0) {
2576 switch (VT.SimpleTy) {
2577 case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0);
2578 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
2579 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
2580 case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0);
2581 case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0);
2582 case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0);
2583 case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0);
2584 default: return Register();
2585 }
2586}
2587
2588// FastEmit functions for ISD::FP_EXTEND.
2589
2590Register fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
2591 if (RetVT.SimpleTy != MVT::f64)
2592 return Register();
2593 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2594 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTDS, RC: &ARM::DPRRegClass, Op0);
2595 }
2596 return Register();
2597}
2598
2599Register fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
2600 if (RetVT.SimpleTy != MVT::v4f32)
2601 return Register();
2602 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2f, RC: &ARM::QPRRegClass, Op0);
2603}
2604
2605Register fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
2606 switch (VT.SimpleTy) {
2607 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
2608 case MVT::v4f16: return fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
2609 default: return Register();
2610 }
2611}
2612
2613// FastEmit functions for ISD::FP_ROUND.
2614
2615Register fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
2616 if (RetVT.SimpleTy != MVT::f32)
2617 return Register();
2618 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2619 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTSD, RC: &ARM::SPRRegClass, Op0);
2620 }
2621 return Register();
2622}
2623
2624Register fastEmit_ISD_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
2625 if (RetVT.SimpleTy != MVT::v4f16)
2626 return Register();
2627 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2h, RC: &ARM::DPRRegClass, Op0);
2628}
2629
2630Register fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
2631 switch (VT.SimpleTy) {
2632 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
2633 case MVT::v4f32: return fastEmit_ISD_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
2634 default: return Register();
2635 }
2636}
2637
2638// FastEmit functions for ISD::FP_TO_SINT.
2639
2640Register fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2641 if (RetVT.SimpleTy != MVT::v4i16)
2642 return Register();
2643 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2644 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2sd, RC: &ARM::DPRRegClass, Op0);
2645 }
2646 return Register();
2647}
2648
2649Register fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2650 if (RetVT.SimpleTy != MVT::v8i16)
2651 return Register();
2652 if ((Subtarget->hasMVEFloatOps())) {
2653 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTs16f16z, RC: &ARM::MQPRRegClass, Op0);
2654 }
2655 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2656 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2sq, RC: &ARM::QPRRegClass, Op0);
2657 }
2658 return Register();
2659}
2660
2661Register fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2662 if (RetVT.SimpleTy != MVT::v2i32)
2663 return Register();
2664 if ((Subtarget->hasNEON())) {
2665 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2sd, RC: &ARM::DPRRegClass, Op0);
2666 }
2667 return Register();
2668}
2669
2670Register fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2671 if (RetVT.SimpleTy != MVT::v4i32)
2672 return Register();
2673 if ((Subtarget->hasMVEFloatOps())) {
2674 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTs32f32z, RC: &ARM::MQPRRegClass, Op0);
2675 }
2676 if ((Subtarget->hasNEON())) {
2677 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2sq, RC: &ARM::QPRRegClass, Op0);
2678 }
2679 return Register();
2680}
2681
2682Register fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
2683 switch (VT.SimpleTy) {
2684 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
2685 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
2686 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
2687 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
2688 default: return Register();
2689 }
2690}
2691
2692// FastEmit functions for ISD::FP_TO_UINT.
2693
2694Register fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2695 if (RetVT.SimpleTy != MVT::v4i16)
2696 return Register();
2697 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2698 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2ud, RC: &ARM::DPRRegClass, Op0);
2699 }
2700 return Register();
2701}
2702
2703Register fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2704 if (RetVT.SimpleTy != MVT::v8i16)
2705 return Register();
2706 if ((Subtarget->hasMVEFloatOps())) {
2707 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTu16f16z, RC: &ARM::MQPRRegClass, Op0);
2708 }
2709 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2710 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTh2uq, RC: &ARM::QPRRegClass, Op0);
2711 }
2712 return Register();
2713}
2714
2715Register fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2716 if (RetVT.SimpleTy != MVT::v2i32)
2717 return Register();
2718 if ((Subtarget->hasNEON())) {
2719 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2ud, RC: &ARM::DPRRegClass, Op0);
2720 }
2721 return Register();
2722}
2723
2724Register fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2725 if (RetVT.SimpleTy != MVT::v4i32)
2726 return Register();
2727 if ((Subtarget->hasMVEFloatOps())) {
2728 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTu32f32z, RC: &ARM::MQPRRegClass, Op0);
2729 }
2730 if ((Subtarget->hasNEON())) {
2731 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTf2uq, RC: &ARM::QPRRegClass, Op0);
2732 }
2733 return Register();
2734}
2735
2736Register fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
2737 switch (VT.SimpleTy) {
2738 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
2739 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
2740 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
2741 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
2742 default: return Register();
2743 }
2744}
2745
2746// FastEmit functions for ISD::FRINT.
2747
2748Register fastEmit_ISD_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
2749 if (RetVT.SimpleTy != MVT::f16)
2750 return Register();
2751 if ((Subtarget->hasFullFP16())) {
2752 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXH, RC: &ARM::HPRRegClass, Op0);
2753 }
2754 return Register();
2755}
2756
2757Register fastEmit_ISD_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
2758 if (RetVT.SimpleTy != MVT::f32)
2759 return Register();
2760 if ((Subtarget->hasFPARMv8Base())) {
2761 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXS, RC: &ARM::SPRRegClass, Op0);
2762 }
2763 return Register();
2764}
2765
2766Register fastEmit_ISD_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
2767 if (RetVT.SimpleTy != MVT::f64)
2768 return Register();
2769 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2770 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXD, RC: &ARM::DPRRegClass, Op0);
2771 }
2772 return Register();
2773}
2774
2775Register fastEmit_ISD_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2776 if (RetVT.SimpleTy != MVT::v4f16)
2777 return Register();
2778 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2779 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNDh, RC: &ARM::DPRRegClass, Op0);
2780 }
2781 return Register();
2782}
2783
2784Register fastEmit_ISD_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2785 if (RetVT.SimpleTy != MVT::v8f16)
2786 return Register();
2787 if ((Subtarget->hasMVEFloatOps())) {
2788 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16X, RC: &ARM::MQPRRegClass, Op0);
2789 }
2790 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2791 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNQh, RC: &ARM::QPRRegClass, Op0);
2792 }
2793 return Register();
2794}
2795
2796Register fastEmit_ISD_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2797 if (RetVT.SimpleTy != MVT::v2f32)
2798 return Register();
2799 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2800 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNDf, RC: &ARM::DPRRegClass, Op0);
2801 }
2802 return Register();
2803}
2804
2805Register fastEmit_ISD_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2806 if (RetVT.SimpleTy != MVT::v4f32)
2807 return Register();
2808 if ((Subtarget->hasMVEFloatOps())) {
2809 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32X, RC: &ARM::MQPRRegClass, Op0);
2810 }
2811 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2812 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNQf, RC: &ARM::QPRRegClass, Op0);
2813 }
2814 return Register();
2815}
2816
2817Register fastEmit_ISD_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
2818 switch (VT.SimpleTy) {
2819 case MVT::f16: return fastEmit_ISD_FRINT_MVT_f16_r(RetVT, Op0);
2820 case MVT::f32: return fastEmit_ISD_FRINT_MVT_f32_r(RetVT, Op0);
2821 case MVT::f64: return fastEmit_ISD_FRINT_MVT_f64_r(RetVT, Op0);
2822 case MVT::v4f16: return fastEmit_ISD_FRINT_MVT_v4f16_r(RetVT, Op0);
2823 case MVT::v8f16: return fastEmit_ISD_FRINT_MVT_v8f16_r(RetVT, Op0);
2824 case MVT::v2f32: return fastEmit_ISD_FRINT_MVT_v2f32_r(RetVT, Op0);
2825 case MVT::v4f32: return fastEmit_ISD_FRINT_MVT_v4f32_r(RetVT, Op0);
2826 default: return Register();
2827 }
2828}
2829
2830// FastEmit functions for ISD::FROUND.
2831
2832Register fastEmit_ISD_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
2833 if (RetVT.SimpleTy != MVT::f16)
2834 return Register();
2835 if ((Subtarget->hasFullFP16())) {
2836 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAH, RC: &ARM::HPRRegClass, Op0);
2837 }
2838 return Register();
2839}
2840
2841Register fastEmit_ISD_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
2842 if (RetVT.SimpleTy != MVT::f32)
2843 return Register();
2844 if ((Subtarget->hasFPARMv8Base())) {
2845 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAS, RC: &ARM::SPRRegClass, Op0);
2846 }
2847 return Register();
2848}
2849
2850Register fastEmit_ISD_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
2851 if (RetVT.SimpleTy != MVT::f64)
2852 return Register();
2853 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2854 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAD, RC: &ARM::DPRRegClass, Op0);
2855 }
2856 return Register();
2857}
2858
2859Register fastEmit_ISD_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
2860 if (RetVT.SimpleTy != MVT::v4f16)
2861 return Register();
2862 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2863 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANDh, RC: &ARM::DPRRegClass, Op0);
2864 }
2865 return Register();
2866}
2867
2868Register fastEmit_ISD_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
2869 if (RetVT.SimpleTy != MVT::v8f16)
2870 return Register();
2871 if ((Subtarget->hasMVEFloatOps())) {
2872 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16A, RC: &ARM::MQPRRegClass, Op0);
2873 }
2874 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2875 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANQh, RC: &ARM::QPRRegClass, Op0);
2876 }
2877 return Register();
2878}
2879
2880Register fastEmit_ISD_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
2881 if (RetVT.SimpleTy != MVT::v2f32)
2882 return Register();
2883 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2884 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANDf, RC: &ARM::DPRRegClass, Op0);
2885 }
2886 return Register();
2887}
2888
2889Register fastEmit_ISD_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
2890 if (RetVT.SimpleTy != MVT::v4f32)
2891 return Register();
2892 if ((Subtarget->hasMVEFloatOps())) {
2893 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32A, RC: &ARM::MQPRRegClass, Op0);
2894 }
2895 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2896 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANQf, RC: &ARM::QPRRegClass, Op0);
2897 }
2898 return Register();
2899}
2900
2901Register fastEmit_ISD_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
2902 switch (VT.SimpleTy) {
2903 case MVT::f16: return fastEmit_ISD_FROUND_MVT_f16_r(RetVT, Op0);
2904 case MVT::f32: return fastEmit_ISD_FROUND_MVT_f32_r(RetVT, Op0);
2905 case MVT::f64: return fastEmit_ISD_FROUND_MVT_f64_r(RetVT, Op0);
2906 case MVT::v4f16: return fastEmit_ISD_FROUND_MVT_v4f16_r(RetVT, Op0);
2907 case MVT::v8f16: return fastEmit_ISD_FROUND_MVT_v8f16_r(RetVT, Op0);
2908 case MVT::v2f32: return fastEmit_ISD_FROUND_MVT_v2f32_r(RetVT, Op0);
2909 case MVT::v4f32: return fastEmit_ISD_FROUND_MVT_v4f32_r(RetVT, Op0);
2910 default: return Register();
2911 }
2912}
2913
2914// FastEmit functions for ISD::FROUNDEVEN.
2915
2916Register fastEmit_ISD_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
2917 if (RetVT.SimpleTy != MVT::f16)
2918 return Register();
2919 if ((Subtarget->hasFullFP16())) {
2920 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNH, RC: &ARM::HPRRegClass, Op0);
2921 }
2922 return Register();
2923}
2924
2925Register fastEmit_ISD_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
2926 if (RetVT.SimpleTy != MVT::f32)
2927 return Register();
2928 if ((Subtarget->hasFPARMv8Base())) {
2929 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNS, RC: &ARM::SPRRegClass, Op0);
2930 }
2931 return Register();
2932}
2933
2934Register fastEmit_ISD_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
2935 if (RetVT.SimpleTy != MVT::f64)
2936 return Register();
2937 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2938 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTND, RC: &ARM::DPRRegClass, Op0);
2939 }
2940 return Register();
2941}
2942
2943Register fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
2944 if (RetVT.SimpleTy != MVT::v4f16)
2945 return Register();
2946 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2947 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNDh, RC: &ARM::DPRRegClass, Op0);
2948 }
2949 return Register();
2950}
2951
2952Register fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
2953 if (RetVT.SimpleTy != MVT::v8f16)
2954 return Register();
2955 if ((Subtarget->hasMVEFloatOps())) {
2956 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16N, RC: &ARM::MQPRRegClass, Op0);
2957 }
2958 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2959 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNQh, RC: &ARM::QPRRegClass, Op0);
2960 }
2961 return Register();
2962}
2963
2964Register fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
2965 if (RetVT.SimpleTy != MVT::v2f32)
2966 return Register();
2967 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2968 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNDf, RC: &ARM::DPRRegClass, Op0);
2969 }
2970 return Register();
2971}
2972
2973Register fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
2974 if (RetVT.SimpleTy != MVT::v4f32)
2975 return Register();
2976 if ((Subtarget->hasMVEFloatOps())) {
2977 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32N, RC: &ARM::MQPRRegClass, Op0);
2978 }
2979 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
2980 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNQf, RC: &ARM::QPRRegClass, Op0);
2981 }
2982 return Register();
2983}
2984
2985Register fastEmit_ISD_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
2986 switch (VT.SimpleTy) {
2987 case MVT::f16: return fastEmit_ISD_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
2988 case MVT::f32: return fastEmit_ISD_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
2989 case MVT::f64: return fastEmit_ISD_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
2990 case MVT::v4f16: return fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
2991 case MVT::v8f16: return fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
2992 case MVT::v2f32: return fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
2993 case MVT::v4f32: return fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
2994 default: return Register();
2995 }
2996}
2997
2998// FastEmit functions for ISD::FSQRT.
2999
3000Register fastEmit_ISD_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
3001 if (RetVT.SimpleTy != MVT::f16)
3002 return Register();
3003 if ((Subtarget->hasFullFP16())) {
3004 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTH, RC: &ARM::HPRRegClass, Op0);
3005 }
3006 return Register();
3007}
3008
3009Register fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
3010 if (RetVT.SimpleTy != MVT::f32)
3011 return Register();
3012 if ((Subtarget->hasVFP2Base())) {
3013 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTS, RC: &ARM::SPRRegClass, Op0);
3014 }
3015 return Register();
3016}
3017
3018Register fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
3019 if (RetVT.SimpleTy != MVT::f64)
3020 return Register();
3021 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3022 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTD, RC: &ARM::DPRRegClass, Op0);
3023 }
3024 return Register();
3025}
3026
3027Register fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
3028 switch (VT.SimpleTy) {
3029 case MVT::f16: return fastEmit_ISD_FSQRT_MVT_f16_r(RetVT, Op0);
3030 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
3031 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
3032 default: return Register();
3033 }
3034}
3035
3036// FastEmit functions for ISD::FTRUNC.
3037
3038Register fastEmit_ISD_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
3039 if (RetVT.SimpleTy != MVT::f16)
3040 return Register();
3041 if ((Subtarget->hasFullFP16())) {
3042 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZH, RC: &ARM::HPRRegClass, Op0);
3043 }
3044 return Register();
3045}
3046
3047Register fastEmit_ISD_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
3048 if (RetVT.SimpleTy != MVT::f32)
3049 return Register();
3050 if ((Subtarget->hasFPARMv8Base())) {
3051 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZS, RC: &ARM::SPRRegClass, Op0);
3052 }
3053 return Register();
3054}
3055
3056Register fastEmit_ISD_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
3057 if (RetVT.SimpleTy != MVT::f64)
3058 return Register();
3059 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3060 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZD, RC: &ARM::DPRRegClass, Op0);
3061 }
3062 return Register();
3063}
3064
3065Register fastEmit_ISD_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
3066 if (RetVT.SimpleTy != MVT::v4f16)
3067 return Register();
3068 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3069 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNDh, RC: &ARM::DPRRegClass, Op0);
3070 }
3071 return Register();
3072}
3073
3074Register fastEmit_ISD_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
3075 if (RetVT.SimpleTy != MVT::v8f16)
3076 return Register();
3077 if ((Subtarget->hasMVEFloatOps())) {
3078 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16Z, RC: &ARM::MQPRRegClass, Op0);
3079 }
3080 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3081 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNQh, RC: &ARM::QPRRegClass, Op0);
3082 }
3083 return Register();
3084}
3085
3086Register fastEmit_ISD_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
3087 if (RetVT.SimpleTy != MVT::v2f32)
3088 return Register();
3089 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3090 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNDf, RC: &ARM::DPRRegClass, Op0);
3091 }
3092 return Register();
3093}
3094
3095Register fastEmit_ISD_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
3096 if (RetVT.SimpleTy != MVT::v4f32)
3097 return Register();
3098 if ((Subtarget->hasMVEFloatOps())) {
3099 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32Z, RC: &ARM::MQPRRegClass, Op0);
3100 }
3101 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3102 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNQf, RC: &ARM::QPRRegClass, Op0);
3103 }
3104 return Register();
3105}
3106
3107Register fastEmit_ISD_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
3108 switch (VT.SimpleTy) {
3109 case MVT::f16: return fastEmit_ISD_FTRUNC_MVT_f16_r(RetVT, Op0);
3110 case MVT::f32: return fastEmit_ISD_FTRUNC_MVT_f32_r(RetVT, Op0);
3111 case MVT::f64: return fastEmit_ISD_FTRUNC_MVT_f64_r(RetVT, Op0);
3112 case MVT::v4f16: return fastEmit_ISD_FTRUNC_MVT_v4f16_r(RetVT, Op0);
3113 case MVT::v8f16: return fastEmit_ISD_FTRUNC_MVT_v8f16_r(RetVT, Op0);
3114 case MVT::v2f32: return fastEmit_ISD_FTRUNC_MVT_v2f32_r(RetVT, Op0);
3115 case MVT::v4f32: return fastEmit_ISD_FTRUNC_MVT_v4f32_r(RetVT, Op0);
3116 default: return Register();
3117 }
3118}
3119
3120// FastEmit functions for ISD::SET_FPENV.
3121
3122Register fastEmit_ISD_SET_FPENV_MVT_i32_r(MVT RetVT, Register Op0) {
3123 if (RetVT.SimpleTy != MVT::isVoid)
3124 return Register();
3125 return fastEmitInst_r(MachineInstOpcode: ARM::VMSR, RC: &ARM::GPRnopcRegClass, Op0);
3126}
3127
3128Register fastEmit_ISD_SET_FPENV_r(MVT VT, MVT RetVT, Register Op0) {
3129 switch (VT.SimpleTy) {
3130 case MVT::i32: return fastEmit_ISD_SET_FPENV_MVT_i32_r(RetVT, Op0);
3131 default: return Register();
3132 }
3133}
3134
3135// FastEmit functions for ISD::SIGN_EXTEND.
3136
3137Register fastEmit_ISD_SIGN_EXTEND_MVT_v8i8_r(MVT RetVT, Register Op0) {
3138 if (RetVT.SimpleTy != MVT::v8i16)
3139 return Register();
3140 if ((Subtarget->hasNEON())) {
3141 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLsv8i16, RC: &ARM::QPRRegClass, Op0);
3142 }
3143 return Register();
3144}
3145
3146Register fastEmit_ISD_SIGN_EXTEND_MVT_v4i16_r(MVT RetVT, Register Op0) {
3147 if (RetVT.SimpleTy != MVT::v4i32)
3148 return Register();
3149 if ((Subtarget->hasNEON())) {
3150 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLsv4i32, RC: &ARM::QPRRegClass, Op0);
3151 }
3152 return Register();
3153}
3154
3155Register fastEmit_ISD_SIGN_EXTEND_MVT_v2i32_r(MVT RetVT, Register Op0) {
3156 if (RetVT.SimpleTy != MVT::v2i64)
3157 return Register();
3158 if ((Subtarget->hasNEON())) {
3159 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLsv2i64, RC: &ARM::QPRRegClass, Op0);
3160 }
3161 return Register();
3162}
3163
3164Register fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3165 switch (VT.SimpleTy) {
3166 case MVT::v8i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i8_r(RetVT, Op0);
3167 case MVT::v4i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i16_r(RetVT, Op0);
3168 case MVT::v2i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i32_r(RetVT, Op0);
3169 default: return Register();
3170 }
3171}
3172
3173// FastEmit functions for ISD::SINT_TO_FP.
3174
3175Register fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
3176 if (RetVT.SimpleTy != MVT::v4f16)
3177 return Register();
3178 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3179 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2hd, RC: &ARM::DPRRegClass, Op0);
3180 }
3181 return Register();
3182}
3183
3184Register fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
3185 if (RetVT.SimpleTy != MVT::v8f16)
3186 return Register();
3187 if ((Subtarget->hasMVEFloatOps())) {
3188 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf16s16n, RC: &ARM::MQPRRegClass, Op0);
3189 }
3190 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3191 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2hq, RC: &ARM::QPRRegClass, Op0);
3192 }
3193 return Register();
3194}
3195
3196Register fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
3197 if (RetVT.SimpleTy != MVT::v2f32)
3198 return Register();
3199 if ((Subtarget->hasNEON())) {
3200 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2fd, RC: &ARM::DPRRegClass, Op0);
3201 }
3202 return Register();
3203}
3204
3205Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
3206 if (RetVT.SimpleTy != MVT::v4f32)
3207 return Register();
3208 if ((Subtarget->hasMVEFloatOps())) {
3209 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf32s32n, RC: &ARM::MQPRRegClass, Op0);
3210 }
3211 if ((Subtarget->hasNEON())) {
3212 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTs2fq, RC: &ARM::QPRRegClass, Op0);
3213 }
3214 return Register();
3215}
3216
3217Register fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3218 switch (VT.SimpleTy) {
3219 case MVT::v4i16: return fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
3220 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3221 case MVT::v2i32: return fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
3222 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3223 default: return Register();
3224 }
3225}
3226
3227// FastEmit functions for ISD::STRICT_FCEIL.
3228
3229Register fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
3230 if (RetVT.SimpleTy != MVT::f16)
3231 return Register();
3232 if ((Subtarget->hasFullFP16())) {
3233 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPH, RC: &ARM::HPRRegClass, Op0);
3234 }
3235 return Register();
3236}
3237
3238Register fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
3239 if (RetVT.SimpleTy != MVT::f32)
3240 return Register();
3241 if ((Subtarget->hasFPARMv8Base())) {
3242 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPS, RC: &ARM::SPRRegClass, Op0);
3243 }
3244 return Register();
3245}
3246
3247Register fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
3248 if (RetVT.SimpleTy != MVT::f64)
3249 return Register();
3250 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3251 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPD, RC: &ARM::DPRRegClass, Op0);
3252 }
3253 return Register();
3254}
3255
3256Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
3257 if (RetVT.SimpleTy != MVT::v4f16)
3258 return Register();
3259 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3260 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNDh, RC: &ARM::DPRRegClass, Op0);
3261 }
3262 return Register();
3263}
3264
3265Register fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
3266 if (RetVT.SimpleTy != MVT::v8f16)
3267 return Register();
3268 if ((Subtarget->hasMVEFloatOps())) {
3269 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16P, RC: &ARM::MQPRRegClass, Op0);
3270 }
3271 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3272 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNQh, RC: &ARM::QPRRegClass, Op0);
3273 }
3274 return Register();
3275}
3276
3277Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
3278 if (RetVT.SimpleTy != MVT::v2f32)
3279 return Register();
3280 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3281 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNDf, RC: &ARM::DPRRegClass, Op0);
3282 }
3283 return Register();
3284}
3285
3286Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
3287 if (RetVT.SimpleTy != MVT::v4f32)
3288 return Register();
3289 if ((Subtarget->hasMVEFloatOps())) {
3290 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32P, RC: &ARM::MQPRRegClass, Op0);
3291 }
3292 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3293 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTPNQf, RC: &ARM::QPRRegClass, Op0);
3294 }
3295 return Register();
3296}
3297
3298Register fastEmit_ISD_STRICT_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
3299 switch (VT.SimpleTy) {
3300 case MVT::f16: return fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(RetVT, Op0);
3301 case MVT::f32: return fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(RetVT, Op0);
3302 case MVT::f64: return fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(RetVT, Op0);
3303 case MVT::v4f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(RetVT, Op0);
3304 case MVT::v8f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(RetVT, Op0);
3305 case MVT::v2f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(RetVT, Op0);
3306 case MVT::v4f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(RetVT, Op0);
3307 default: return Register();
3308 }
3309}
3310
3311// FastEmit functions for ISD::STRICT_FFLOOR.
3312
3313Register fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
3314 if (RetVT.SimpleTy != MVT::f16)
3315 return Register();
3316 if ((Subtarget->hasFullFP16())) {
3317 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMH, RC: &ARM::HPRRegClass, Op0);
3318 }
3319 return Register();
3320}
3321
3322Register fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
3323 if (RetVT.SimpleTy != MVT::f32)
3324 return Register();
3325 if ((Subtarget->hasFPARMv8Base())) {
3326 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMS, RC: &ARM::SPRRegClass, Op0);
3327 }
3328 return Register();
3329}
3330
3331Register fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
3332 if (RetVT.SimpleTy != MVT::f64)
3333 return Register();
3334 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3335 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMD, RC: &ARM::DPRRegClass, Op0);
3336 }
3337 return Register();
3338}
3339
3340Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
3341 if (RetVT.SimpleTy != MVT::v4f16)
3342 return Register();
3343 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3344 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNDh, RC: &ARM::DPRRegClass, Op0);
3345 }
3346 return Register();
3347}
3348
3349Register fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
3350 if (RetVT.SimpleTy != MVT::v8f16)
3351 return Register();
3352 if ((Subtarget->hasMVEFloatOps())) {
3353 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16M, RC: &ARM::MQPRRegClass, Op0);
3354 }
3355 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3356 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNQh, RC: &ARM::QPRRegClass, Op0);
3357 }
3358 return Register();
3359}
3360
3361Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
3362 if (RetVT.SimpleTy != MVT::v2f32)
3363 return Register();
3364 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3365 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNDf, RC: &ARM::DPRRegClass, Op0);
3366 }
3367 return Register();
3368}
3369
3370Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
3371 if (RetVT.SimpleTy != MVT::v4f32)
3372 return Register();
3373 if ((Subtarget->hasMVEFloatOps())) {
3374 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32M, RC: &ARM::MQPRRegClass, Op0);
3375 }
3376 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3377 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTMNQf, RC: &ARM::QPRRegClass, Op0);
3378 }
3379 return Register();
3380}
3381
3382Register fastEmit_ISD_STRICT_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
3383 switch (VT.SimpleTy) {
3384 case MVT::f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(RetVT, Op0);
3385 case MVT::f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(RetVT, Op0);
3386 case MVT::f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(RetVT, Op0);
3387 case MVT::v4f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(RetVT, Op0);
3388 case MVT::v8f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(RetVT, Op0);
3389 case MVT::v2f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(RetVT, Op0);
3390 case MVT::v4f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(RetVT, Op0);
3391 default: return Register();
3392 }
3393}
3394
3395// FastEmit functions for ISD::STRICT_FNEARBYINT.
3396
3397Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
3398 if (RetVT.SimpleTy != MVT::f16)
3399 return Register();
3400 if ((Subtarget->hasFullFP16())) {
3401 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRH, RC: &ARM::HPRRegClass, Op0);
3402 }
3403 return Register();
3404}
3405
3406Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
3407 if (RetVT.SimpleTy != MVT::f32)
3408 return Register();
3409 if ((Subtarget->hasFPARMv8Base())) {
3410 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRS, RC: &ARM::SPRRegClass, Op0);
3411 }
3412 return Register();
3413}
3414
3415Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
3416 if (RetVT.SimpleTy != MVT::f64)
3417 return Register();
3418 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3419 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTRD, RC: &ARM::DPRRegClass, Op0);
3420 }
3421 return Register();
3422}
3423
3424Register fastEmit_ISD_STRICT_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
3425 switch (VT.SimpleTy) {
3426 case MVT::f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(RetVT, Op0);
3427 case MVT::f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(RetVT, Op0);
3428 case MVT::f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(RetVT, Op0);
3429 default: return Register();
3430 }
3431}
3432
3433// FastEmit functions for ISD::STRICT_FP_EXTEND.
3434
3435Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
3436 if (RetVT.SimpleTy != MVT::f64)
3437 return Register();
3438 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3439 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTDS, RC: &ARM::DPRRegClass, Op0);
3440 }
3441 return Register();
3442}
3443
3444Register fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3445 switch (VT.SimpleTy) {
3446 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0);
3447 default: return Register();
3448 }
3449}
3450
3451// FastEmit functions for ISD::STRICT_FP_ROUND.
3452
3453Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3454 if (RetVT.SimpleTy != MVT::f32)
3455 return Register();
3456 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3457 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTSD, RC: &ARM::SPRRegClass, Op0);
3458 }
3459 return Register();
3460}
3461
3462Register fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
3463 switch (VT.SimpleTy) {
3464 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0);
3465 default: return Register();
3466 }
3467}
3468
3469// FastEmit functions for ISD::STRICT_FRINT.
3470
3471Register fastEmit_ISD_STRICT_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
3472 if (RetVT.SimpleTy != MVT::f16)
3473 return Register();
3474 if ((Subtarget->hasFullFP16())) {
3475 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXH, RC: &ARM::HPRRegClass, Op0);
3476 }
3477 return Register();
3478}
3479
3480Register fastEmit_ISD_STRICT_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
3481 if (RetVT.SimpleTy != MVT::f32)
3482 return Register();
3483 if ((Subtarget->hasFPARMv8Base())) {
3484 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXS, RC: &ARM::SPRRegClass, Op0);
3485 }
3486 return Register();
3487}
3488
3489Register fastEmit_ISD_STRICT_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
3490 if (RetVT.SimpleTy != MVT::f64)
3491 return Register();
3492 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3493 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXD, RC: &ARM::DPRRegClass, Op0);
3494 }
3495 return Register();
3496}
3497
3498Register fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3499 if (RetVT.SimpleTy != MVT::v4f16)
3500 return Register();
3501 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3502 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNDh, RC: &ARM::DPRRegClass, Op0);
3503 }
3504 return Register();
3505}
3506
3507Register fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3508 if (RetVT.SimpleTy != MVT::v8f16)
3509 return Register();
3510 if ((Subtarget->hasMVEFloatOps())) {
3511 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16X, RC: &ARM::MQPRRegClass, Op0);
3512 }
3513 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3514 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNQh, RC: &ARM::QPRRegClass, Op0);
3515 }
3516 return Register();
3517}
3518
3519Register fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3520 if (RetVT.SimpleTy != MVT::v2f32)
3521 return Register();
3522 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3523 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNDf, RC: &ARM::DPRRegClass, Op0);
3524 }
3525 return Register();
3526}
3527
3528Register fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3529 if (RetVT.SimpleTy != MVT::v4f32)
3530 return Register();
3531 if ((Subtarget->hasMVEFloatOps())) {
3532 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32X, RC: &ARM::MQPRRegClass, Op0);
3533 }
3534 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3535 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTXNQf, RC: &ARM::QPRRegClass, Op0);
3536 }
3537 return Register();
3538}
3539
3540Register fastEmit_ISD_STRICT_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
3541 switch (VT.SimpleTy) {
3542 case MVT::f16: return fastEmit_ISD_STRICT_FRINT_MVT_f16_r(RetVT, Op0);
3543 case MVT::f32: return fastEmit_ISD_STRICT_FRINT_MVT_f32_r(RetVT, Op0);
3544 case MVT::f64: return fastEmit_ISD_STRICT_FRINT_MVT_f64_r(RetVT, Op0);
3545 case MVT::v4f16: return fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(RetVT, Op0);
3546 case MVT::v8f16: return fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(RetVT, Op0);
3547 case MVT::v2f32: return fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(RetVT, Op0);
3548 case MVT::v4f32: return fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(RetVT, Op0);
3549 default: return Register();
3550 }
3551}
3552
3553// FastEmit functions for ISD::STRICT_FROUND.
3554
3555Register fastEmit_ISD_STRICT_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
3556 if (RetVT.SimpleTy != MVT::f16)
3557 return Register();
3558 if ((Subtarget->hasFullFP16())) {
3559 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAH, RC: &ARM::HPRRegClass, Op0);
3560 }
3561 return Register();
3562}
3563
3564Register fastEmit_ISD_STRICT_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3565 if (RetVT.SimpleTy != MVT::f32)
3566 return Register();
3567 if ((Subtarget->hasFPARMv8Base())) {
3568 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAS, RC: &ARM::SPRRegClass, Op0);
3569 }
3570 return Register();
3571}
3572
3573Register fastEmit_ISD_STRICT_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3574 if (RetVT.SimpleTy != MVT::f64)
3575 return Register();
3576 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3577 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTAD, RC: &ARM::DPRRegClass, Op0);
3578 }
3579 return Register();
3580}
3581
3582Register fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3583 if (RetVT.SimpleTy != MVT::v4f16)
3584 return Register();
3585 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3586 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANDh, RC: &ARM::DPRRegClass, Op0);
3587 }
3588 return Register();
3589}
3590
3591Register fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
3592 if (RetVT.SimpleTy != MVT::v8f16)
3593 return Register();
3594 if ((Subtarget->hasMVEFloatOps())) {
3595 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16A, RC: &ARM::MQPRRegClass, Op0);
3596 }
3597 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3598 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANQh, RC: &ARM::QPRRegClass, Op0);
3599 }
3600 return Register();
3601}
3602
3603Register fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3604 if (RetVT.SimpleTy != MVT::v2f32)
3605 return Register();
3606 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3607 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANDf, RC: &ARM::DPRRegClass, Op0);
3608 }
3609 return Register();
3610}
3611
3612Register fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3613 if (RetVT.SimpleTy != MVT::v4f32)
3614 return Register();
3615 if ((Subtarget->hasMVEFloatOps())) {
3616 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32A, RC: &ARM::MQPRRegClass, Op0);
3617 }
3618 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3619 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTANQf, RC: &ARM::QPRRegClass, Op0);
3620 }
3621 return Register();
3622}
3623
3624Register fastEmit_ISD_STRICT_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
3625 switch (VT.SimpleTy) {
3626 case MVT::f16: return fastEmit_ISD_STRICT_FROUND_MVT_f16_r(RetVT, Op0);
3627 case MVT::f32: return fastEmit_ISD_STRICT_FROUND_MVT_f32_r(RetVT, Op0);
3628 case MVT::f64: return fastEmit_ISD_STRICT_FROUND_MVT_f64_r(RetVT, Op0);
3629 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(RetVT, Op0);
3630 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(RetVT, Op0);
3631 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(RetVT, Op0);
3632 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(RetVT, Op0);
3633 default: return Register();
3634 }
3635}
3636
3637// FastEmit functions for ISD::STRICT_FROUNDEVEN.
3638
3639Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
3640 if (RetVT.SimpleTy != MVT::f16)
3641 return Register();
3642 if ((Subtarget->hasFullFP16())) {
3643 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNH, RC: &ARM::HPRRegClass, Op0);
3644 }
3645 return Register();
3646}
3647
3648Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
3649 if (RetVT.SimpleTy != MVT::f32)
3650 return Register();
3651 if ((Subtarget->hasFPARMv8Base())) {
3652 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNS, RC: &ARM::SPRRegClass, Op0);
3653 }
3654 return Register();
3655}
3656
3657Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
3658 if (RetVT.SimpleTy != MVT::f64)
3659 return Register();
3660 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3661 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTND, RC: &ARM::DPRRegClass, Op0);
3662 }
3663 return Register();
3664}
3665
3666Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
3667 if (RetVT.SimpleTy != MVT::v4f16)
3668 return Register();
3669 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3670 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNDh, RC: &ARM::DPRRegClass, Op0);
3671 }
3672 return Register();
3673}
3674
3675Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
3676 if (RetVT.SimpleTy != MVT::v8f16)
3677 return Register();
3678 if ((Subtarget->hasMVEFloatOps())) {
3679 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16N, RC: &ARM::MQPRRegClass, Op0);
3680 }
3681 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3682 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNQh, RC: &ARM::QPRRegClass, Op0);
3683 }
3684 return Register();
3685}
3686
3687Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
3688 if (RetVT.SimpleTy != MVT::v2f32)
3689 return Register();
3690 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3691 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNDf, RC: &ARM::DPRRegClass, Op0);
3692 }
3693 return Register();
3694}
3695
3696Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
3697 if (RetVT.SimpleTy != MVT::v4f32)
3698 return Register();
3699 if ((Subtarget->hasMVEFloatOps())) {
3700 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32N, RC: &ARM::MQPRRegClass, Op0);
3701 }
3702 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3703 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTNNQf, RC: &ARM::QPRRegClass, Op0);
3704 }
3705 return Register();
3706}
3707
3708Register fastEmit_ISD_STRICT_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
3709 switch (VT.SimpleTy) {
3710 case MVT::f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
3711 case MVT::f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
3712 case MVT::f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
3713 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
3714 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
3715 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
3716 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
3717 default: return Register();
3718 }
3719}
3720
3721// FastEmit functions for ISD::STRICT_FSQRT.
3722
3723Register fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
3724 if (RetVT.SimpleTy != MVT::f16)
3725 return Register();
3726 if ((Subtarget->hasFullFP16())) {
3727 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTH, RC: &ARM::HPRRegClass, Op0);
3728 }
3729 return Register();
3730}
3731
3732Register fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
3733 if (RetVT.SimpleTy != MVT::f32)
3734 return Register();
3735 if ((Subtarget->hasVFP2Base())) {
3736 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTS, RC: &ARM::SPRRegClass, Op0);
3737 }
3738 return Register();
3739}
3740
3741Register fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
3742 if (RetVT.SimpleTy != MVT::f64)
3743 return Register();
3744 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3745 return fastEmitInst_r(MachineInstOpcode: ARM::VSQRTD, RC: &ARM::DPRRegClass, Op0);
3746 }
3747 return Register();
3748}
3749
3750Register fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
3751 switch (VT.SimpleTy) {
3752 case MVT::f16: return fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(RetVT, Op0);
3753 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0);
3754 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0);
3755 default: return Register();
3756 }
3757}
3758
3759// FastEmit functions for ISD::STRICT_FTRUNC.
3760
3761Register fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
3762 if (RetVT.SimpleTy != MVT::f16)
3763 return Register();
3764 if ((Subtarget->hasFullFP16())) {
3765 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZH, RC: &ARM::HPRRegClass, Op0);
3766 }
3767 return Register();
3768}
3769
3770Register fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
3771 if (RetVT.SimpleTy != MVT::f32)
3772 return Register();
3773 if ((Subtarget->hasFPARMv8Base())) {
3774 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZS, RC: &ARM::SPRRegClass, Op0);
3775 }
3776 return Register();
3777}
3778
3779Register fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
3780 if (RetVT.SimpleTy != MVT::f64)
3781 return Register();
3782 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
3783 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZD, RC: &ARM::DPRRegClass, Op0);
3784 }
3785 return Register();
3786}
3787
3788Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
3789 if (RetVT.SimpleTy != MVT::v4f16)
3790 return Register();
3791 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3792 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNDh, RC: &ARM::DPRRegClass, Op0);
3793 }
3794 return Register();
3795}
3796
3797Register fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
3798 if (RetVT.SimpleTy != MVT::v8f16)
3799 return Register();
3800 if ((Subtarget->hasMVEFloatOps())) {
3801 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf16Z, RC: &ARM::MQPRRegClass, Op0);
3802 }
3803 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3804 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNQh, RC: &ARM::QPRRegClass, Op0);
3805 }
3806 return Register();
3807}
3808
3809Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
3810 if (RetVT.SimpleTy != MVT::v2f32)
3811 return Register();
3812 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3813 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNDf, RC: &ARM::DPRRegClass, Op0);
3814 }
3815 return Register();
3816}
3817
3818Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
3819 if (RetVT.SimpleTy != MVT::v4f32)
3820 return Register();
3821 if ((Subtarget->hasMVEFloatOps())) {
3822 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VRINTf32Z, RC: &ARM::MQPRRegClass, Op0);
3823 }
3824 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
3825 return fastEmitInst_r(MachineInstOpcode: ARM::VRINTZNQf, RC: &ARM::QPRRegClass, Op0);
3826 }
3827 return Register();
3828}
3829
3830Register fastEmit_ISD_STRICT_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
3831 switch (VT.SimpleTy) {
3832 case MVT::f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(RetVT, Op0);
3833 case MVT::f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(RetVT, Op0);
3834 case MVT::f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(RetVT, Op0);
3835 case MVT::v4f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(RetVT, Op0);
3836 case MVT::v8f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(RetVT, Op0);
3837 case MVT::v2f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(RetVT, Op0);
3838 case MVT::v4f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(RetVT, Op0);
3839 default: return Register();
3840 }
3841}
3842
3843// FastEmit functions for ISD::TRUNCATE.
3844
3845Register fastEmit_ISD_TRUNCATE_MVT_v8i16_r(MVT RetVT, Register Op0) {
3846 if (RetVT.SimpleTy != MVT::v8i8)
3847 return Register();
3848 if ((Subtarget->hasNEON())) {
3849 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVNv8i8, RC: &ARM::DPRRegClass, Op0);
3850 }
3851 return Register();
3852}
3853
3854Register fastEmit_ISD_TRUNCATE_MVT_v4i32_r(MVT RetVT, Register Op0) {
3855 if (RetVT.SimpleTy != MVT::v4i16)
3856 return Register();
3857 if ((Subtarget->hasNEON())) {
3858 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVNv4i16, RC: &ARM::DPRRegClass, Op0);
3859 }
3860 return Register();
3861}
3862
3863Register fastEmit_ISD_TRUNCATE_MVT_v2i64_r(MVT RetVT, Register Op0) {
3864 if (RetVT.SimpleTy != MVT::v2i32)
3865 return Register();
3866 if ((Subtarget->hasNEON())) {
3867 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVNv2i32, RC: &ARM::DPRRegClass, Op0);
3868 }
3869 return Register();
3870}
3871
3872Register fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, Register Op0) {
3873 switch (VT.SimpleTy) {
3874 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i16_r(RetVT, Op0);
3875 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_MVT_v4i32_r(RetVT, Op0);
3876 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_MVT_v2i64_r(RetVT, Op0);
3877 default: return Register();
3878 }
3879}
3880
3881// FastEmit functions for ISD::UINT_TO_FP.
3882
3883Register fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
3884 if (RetVT.SimpleTy != MVT::v4f16)
3885 return Register();
3886 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3887 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2hd, RC: &ARM::DPRRegClass, Op0);
3888 }
3889 return Register();
3890}
3891
3892Register fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
3893 if (RetVT.SimpleTy != MVT::v8f16)
3894 return Register();
3895 if ((Subtarget->hasMVEFloatOps())) {
3896 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf16u16n, RC: &ARM::MQPRRegClass, Op0);
3897 }
3898 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3899 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2hq, RC: &ARM::QPRRegClass, Op0);
3900 }
3901 return Register();
3902}
3903
3904Register fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
3905 if (RetVT.SimpleTy != MVT::v2f32)
3906 return Register();
3907 if ((Subtarget->hasNEON())) {
3908 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2fd, RC: &ARM::DPRRegClass, Op0);
3909 }
3910 return Register();
3911}
3912
3913Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
3914 if (RetVT.SimpleTy != MVT::v4f32)
3915 return Register();
3916 if ((Subtarget->hasMVEFloatOps())) {
3917 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VCVTf32u32n, RC: &ARM::MQPRRegClass, Op0);
3918 }
3919 if ((Subtarget->hasNEON())) {
3920 return fastEmitInst_r(MachineInstOpcode: ARM::VCVTu2fq, RC: &ARM::QPRRegClass, Op0);
3921 }
3922 return Register();
3923}
3924
3925Register fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
3926 switch (VT.SimpleTy) {
3927 case MVT::v4i16: return fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
3928 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
3929 case MVT::v2i32: return fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
3930 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
3931 default: return Register();
3932 }
3933}
3934
3935// FastEmit functions for ISD::VECREDUCE_ADD.
3936
3937Register fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(MVT RetVT, Register Op0) {
3938 if (RetVT.SimpleTy != MVT::i32)
3939 return Register();
3940 if ((Subtarget->hasMVEIntegerOps())) {
3941 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu8no_acc, RC: &ARM::tGPREvenRegClass, Op0);
3942 }
3943 return Register();
3944}
3945
3946Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(MVT RetVT, Register Op0) {
3947 if (RetVT.SimpleTy != MVT::i32)
3948 return Register();
3949 if ((Subtarget->hasMVEIntegerOps())) {
3950 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu16no_acc, RC: &ARM::tGPREvenRegClass, Op0);
3951 }
3952 return Register();
3953}
3954
3955Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(MVT RetVT, Register Op0) {
3956 if (RetVT.SimpleTy != MVT::i32)
3957 return Register();
3958 if ((Subtarget->hasMVEIntegerOps())) {
3959 return fastEmitInst_r(MachineInstOpcode: ARM::MVE_VADDVu32no_acc, RC: &ARM::tGPREvenRegClass, Op0);
3960 }
3961 return Register();
3962}
3963
3964Register fastEmit_ISD_VECREDUCE_ADD_r(MVT VT, MVT RetVT, Register Op0) {
3965 switch (VT.SimpleTy) {
3966 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(RetVT, Op0);
3967 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(RetVT, Op0);
3968 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(RetVT, Op0);
3969 default: return Register();
3970 }
3971}
3972
3973// FastEmit functions for ISD::ZERO_EXTEND.
3974
3975Register fastEmit_ISD_ZERO_EXTEND_MVT_v8i8_r(MVT RetVT, Register Op0) {
3976 if (RetVT.SimpleTy != MVT::v8i16)
3977 return Register();
3978 if ((Subtarget->hasNEON())) {
3979 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv8i16, RC: &ARM::QPRRegClass, Op0);
3980 }
3981 return Register();
3982}
3983
3984Register fastEmit_ISD_ZERO_EXTEND_MVT_v4i16_r(MVT RetVT, Register Op0) {
3985 if (RetVT.SimpleTy != MVT::v4i32)
3986 return Register();
3987 if ((Subtarget->hasNEON())) {
3988 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv4i32, RC: &ARM::QPRRegClass, Op0);
3989 }
3990 return Register();
3991}
3992
3993Register fastEmit_ISD_ZERO_EXTEND_MVT_v2i32_r(MVT RetVT, Register Op0) {
3994 if (RetVT.SimpleTy != MVT::v2i64)
3995 return Register();
3996 if ((Subtarget->hasNEON())) {
3997 return fastEmitInst_r(MachineInstOpcode: ARM::VMOVLuv2i64, RC: &ARM::QPRRegClass, Op0);
3998 }
3999 return Register();
4000}
4001
4002Register fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
4003 switch (VT.SimpleTy) {
4004 case MVT::v8i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i8_r(RetVT, Op0);
4005 case MVT::v4i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v4i16_r(RetVT, Op0);
4006 case MVT::v2i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v2i32_r(RetVT, Op0);
4007 default: return Register();
4008 }
4009}
4010
4011// Top-level FastEmit function.
4012
4013Register fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, Register Op0) override {
4014 switch (Opcode) {
4015 case ARMISD::CALL: return fastEmit_ARMISD_CALL_r(VT, RetVT, Op0);
4016 case ARMISD::CALL_NOLINK: return fastEmit_ARMISD_CALL_NOLINK_r(VT, RetVT, Op0);
4017 case ARMISD::CALL_PRED: return fastEmit_ARMISD_CALL_PRED_r(VT, RetVT, Op0);
4018 case ARMISD::CMPFPEw0: return fastEmit_ARMISD_CMPFPEw0_r(VT, RetVT, Op0);
4019 case ARMISD::CMPFPw0: return fastEmit_ARMISD_CMPFPw0_r(VT, RetVT, Op0);
4020 case ARMISD::VADDVs: return fastEmit_ARMISD_VADDVs_r(VT, RetVT, Op0);
4021 case ARMISD::VADDVu: return fastEmit_ARMISD_VADDVu_r(VT, RetVT, Op0);
4022 case ARMISD::VDUP: return fastEmit_ARMISD_VDUP_r(VT, RetVT, Op0);
4023 case ARMISD::VMOVSR: return fastEmit_ARMISD_VMOVSR_r(VT, RetVT, Op0);
4024 case ARMISD::VMOVhr: return fastEmit_ARMISD_VMOVhr_r(VT, RetVT, Op0);
4025 case ARMISD::VMOVrh: return fastEmit_ARMISD_VMOVrh_r(VT, RetVT, Op0);
4026 case ARMISD::VREV16: return fastEmit_ARMISD_VREV16_r(VT, RetVT, Op0);
4027 case ARMISD::VREV32: return fastEmit_ARMISD_VREV32_r(VT, RetVT, Op0);
4028 case ARMISD::VREV64: return fastEmit_ARMISD_VREV64_r(VT, RetVT, Op0);
4029 case ARMISD::WIN__DBZCHK: return fastEmit_ARMISD_WIN__DBZCHK_r(VT, RetVT, Op0);
4030 case ARMISD::WLSSETUP: return fastEmit_ARMISD_WLSSETUP_r(VT, RetVT, Op0);
4031 case ARMISD::tSECALL: return fastEmit_ARMISD_tSECALL_r(VT, RetVT, Op0);
4032 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
4033 case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0);
4034 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
4035 case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0);
4036 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0);
4037 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
4038 case ISD::CTLS: return fastEmit_ISD_CTLS_r(VT, RetVT, Op0);
4039 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
4040 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
4041 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
4042 case ISD::FCEIL: return fastEmit_ISD_FCEIL_r(VT, RetVT, Op0);
4043 case ISD::FFLOOR: return fastEmit_ISD_FFLOOR_r(VT, RetVT, Op0);
4044 case ISD::FNEARBYINT: return fastEmit_ISD_FNEARBYINT_r(VT, RetVT, Op0);
4045 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
4046 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
4047 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
4048 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
4049 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
4050 case ISD::FRINT: return fastEmit_ISD_FRINT_r(VT, RetVT, Op0);
4051 case ISD::FROUND: return fastEmit_ISD_FROUND_r(VT, RetVT, Op0);
4052 case ISD::FROUNDEVEN: return fastEmit_ISD_FROUNDEVEN_r(VT, RetVT, Op0);
4053 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
4054 case ISD::FTRUNC: return fastEmit_ISD_FTRUNC_r(VT, RetVT, Op0);
4055 case ISD::SET_FPENV: return fastEmit_ISD_SET_FPENV_r(VT, RetVT, Op0);
4056 case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0);
4057 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
4058 case ISD::STRICT_FCEIL: return fastEmit_ISD_STRICT_FCEIL_r(VT, RetVT, Op0);
4059 case ISD::STRICT_FFLOOR: return fastEmit_ISD_STRICT_FFLOOR_r(VT, RetVT, Op0);
4060 case ISD::STRICT_FNEARBYINT: return fastEmit_ISD_STRICT_FNEARBYINT_r(VT, RetVT, Op0);
4061 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0);
4062 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0);
4063 case ISD::STRICT_FRINT: return fastEmit_ISD_STRICT_FRINT_r(VT, RetVT, Op0);
4064 case ISD::STRICT_FROUND: return fastEmit_ISD_STRICT_FROUND_r(VT, RetVT, Op0);
4065 case ISD::STRICT_FROUNDEVEN: return fastEmit_ISD_STRICT_FROUNDEVEN_r(VT, RetVT, Op0);
4066 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0);
4067 case ISD::STRICT_FTRUNC: return fastEmit_ISD_STRICT_FTRUNC_r(VT, RetVT, Op0);
4068 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
4069 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
4070 case ISD::VECREDUCE_ADD: return fastEmit_ISD_VECREDUCE_ADD_r(VT, RetVT, Op0);
4071 case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0);
4072 default: return Register();
4073 }
4074}
4075
4076// FastEmit functions for ARMISD::CMP.
4077
4078Register fastEmit_ARMISD_CMP_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4079 if (RetVT.SimpleTy != MVT::i32)
4080 return Register();
4081 if ((Subtarget->isThumb2())) {
4082 return fastEmitInst_rr(MachineInstOpcode: ARM::t2CMPrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4083 }
4084 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
4085 return fastEmitInst_rr(MachineInstOpcode: ARM::tCMPr, RC: &ARM::tGPRRegClass, Op0, Op1);
4086 }
4087 if ((!Subtarget->isThumb())) {
4088 return fastEmitInst_rr(MachineInstOpcode: ARM::CMPrr, RC: &ARM::GPRRegClass, Op0, Op1);
4089 }
4090 return Register();
4091}
4092
4093Register fastEmit_ARMISD_CMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4094 switch (VT.SimpleTy) {
4095 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_rr(RetVT, Op0, Op1);
4096 default: return Register();
4097 }
4098}
4099
4100// FastEmit functions for ARMISD::CMPFP.
4101
4102Register fastEmit_ARMISD_CMPFP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
4103 if (RetVT.SimpleTy != MVT::i32)
4104 return Register();
4105 if ((Subtarget->hasFullFP16())) {
4106 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPH, RC: &ARM::HPRRegClass, Op0, Op1);
4107 }
4108 return Register();
4109}
4110
4111Register fastEmit_ARMISD_CMPFP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
4112 if (RetVT.SimpleTy != MVT::i32)
4113 return Register();
4114 if ((Subtarget->hasVFP2Base())) {
4115 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPS, RC: &ARM::SPRRegClass, Op0, Op1);
4116 }
4117 return Register();
4118}
4119
4120Register fastEmit_ARMISD_CMPFP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
4121 if (RetVT.SimpleTy != MVT::i32)
4122 return Register();
4123 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
4124 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPD, RC: &ARM::DPRRegClass, Op0, Op1);
4125 }
4126 return Register();
4127}
4128
4129Register fastEmit_ARMISD_CMPFP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4130 switch (VT.SimpleTy) {
4131 case MVT::f16: return fastEmit_ARMISD_CMPFP_MVT_f16_rr(RetVT, Op0, Op1);
4132 case MVT::f32: return fastEmit_ARMISD_CMPFP_MVT_f32_rr(RetVT, Op0, Op1);
4133 case MVT::f64: return fastEmit_ARMISD_CMPFP_MVT_f64_rr(RetVT, Op0, Op1);
4134 default: return Register();
4135 }
4136}
4137
4138// FastEmit functions for ARMISD::CMPFPE.
4139
4140Register fastEmit_ARMISD_CMPFPE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
4141 if (RetVT.SimpleTy != MVT::i32)
4142 return Register();
4143 if ((Subtarget->hasFullFP16())) {
4144 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPEH, RC: &ARM::HPRRegClass, Op0, Op1);
4145 }
4146 return Register();
4147}
4148
4149Register fastEmit_ARMISD_CMPFPE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
4150 if (RetVT.SimpleTy != MVT::i32)
4151 return Register();
4152 if ((Subtarget->hasVFP2Base())) {
4153 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPES, RC: &ARM::SPRRegClass, Op0, Op1);
4154 }
4155 return Register();
4156}
4157
4158Register fastEmit_ARMISD_CMPFPE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
4159 if (RetVT.SimpleTy != MVT::i32)
4160 return Register();
4161 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
4162 return fastEmitInst_rr(MachineInstOpcode: ARM::VCMPED, RC: &ARM::DPRRegClass, Op0, Op1);
4163 }
4164 return Register();
4165}
4166
4167Register fastEmit_ARMISD_CMPFPE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4168 switch (VT.SimpleTy) {
4169 case MVT::f16: return fastEmit_ARMISD_CMPFPE_MVT_f16_rr(RetVT, Op0, Op1);
4170 case MVT::f32: return fastEmit_ARMISD_CMPFPE_MVT_f32_rr(RetVT, Op0, Op1);
4171 case MVT::f64: return fastEmit_ARMISD_CMPFPE_MVT_f64_rr(RetVT, Op0, Op1);
4172 default: return Register();
4173 }
4174}
4175
4176// FastEmit functions for ARMISD::CMPZ.
4177
4178Register fastEmit_ARMISD_CMPZ_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4179 if (RetVT.SimpleTy != MVT::i32)
4180 return Register();
4181 if ((Subtarget->isThumb2())) {
4182 return fastEmitInst_rr(MachineInstOpcode: ARM::t2CMPrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4183 }
4184 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
4185 return fastEmitInst_rr(MachineInstOpcode: ARM::tCMPr, RC: &ARM::tGPRRegClass, Op0, Op1);
4186 }
4187 if ((!Subtarget->isThumb())) {
4188 return fastEmitInst_rr(MachineInstOpcode: ARM::CMPrr, RC: &ARM::GPRRegClass, Op0, Op1);
4189 }
4190 return Register();
4191}
4192
4193Register fastEmit_ARMISD_CMPZ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4194 switch (VT.SimpleTy) {
4195 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_rr(RetVT, Op0, Op1);
4196 default: return Register();
4197 }
4198}
4199
4200// FastEmit functions for ARMISD::EH_SJLJ_LONGJMP.
4201
4202Register fastEmit_ARMISD_EH_SJLJ_LONGJMP_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4203 if (RetVT.SimpleTy != MVT::isVoid)
4204 return Register();
4205 if ((Subtarget->isThumb()) && (Subtarget->isTargetWindows())) {
4206 return fastEmitInst_rr(MachineInstOpcode: ARM::tInt_WIN_eh_sjlj_longjmp, RC: &ARM::GPRRegClass, Op0, Op1);
4207 }
4208 if ((!Subtarget->isTargetWindows()) && (Subtarget->isThumb())) {
4209 return fastEmitInst_rr(MachineInstOpcode: ARM::tInt_eh_sjlj_longjmp, RC: &ARM::tGPRRegClass, Op0, Op1);
4210 }
4211 if ((!Subtarget->isThumb())) {
4212 return fastEmitInst_rr(MachineInstOpcode: ARM::Int_eh_sjlj_longjmp, RC: &ARM::GPRRegClass, Op0, Op1);
4213 }
4214 return Register();
4215}
4216
4217Register fastEmit_ARMISD_EH_SJLJ_LONGJMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4218 switch (VT.SimpleTy) {
4219 case MVT::i32: return fastEmit_ARMISD_EH_SJLJ_LONGJMP_MVT_i32_rr(RetVT, Op0, Op1);
4220 default: return Register();
4221 }
4222}
4223
4224// FastEmit functions for ARMISD::EH_SJLJ_SETJMP.
4225
4226Register fastEmit_ARMISD_EH_SJLJ_SETJMP_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4227 if (RetVT.SimpleTy != MVT::i32)
4228 return Register();
4229 if ((Subtarget->isThumb2()) && (!Subtarget->hasVFP2Base())) {
4230 return fastEmitInst_rr(MachineInstOpcode: ARM::t2Int_eh_sjlj_setjmp_nofp, RC: &ARM::tGPRRegClass, Op0, Op1);
4231 }
4232 if ((Subtarget->hasVFP2Base()) && (Subtarget->isThumb2())) {
4233 return fastEmitInst_rr(MachineInstOpcode: ARM::t2Int_eh_sjlj_setjmp, RC: &ARM::tGPRRegClass, Op0, Op1);
4234 }
4235 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
4236 return fastEmitInst_rr(MachineInstOpcode: ARM::tInt_eh_sjlj_setjmp, RC: &ARM::tGPRRegClass, Op0, Op1);
4237 }
4238 if ((!Subtarget->isThumb()) && (!Subtarget->hasVFP2Base())) {
4239 return fastEmitInst_rr(MachineInstOpcode: ARM::Int_eh_sjlj_setjmp_nofp, RC: &ARM::GPRRegClass, Op0, Op1);
4240 }
4241 if ((Subtarget->hasVFP2Base()) && (!Subtarget->isThumb())) {
4242 return fastEmitInst_rr(MachineInstOpcode: ARM::Int_eh_sjlj_setjmp, RC: &ARM::GPRRegClass, Op0, Op1);
4243 }
4244 return Register();
4245}
4246
4247Register fastEmit_ARMISD_EH_SJLJ_SETJMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4248 switch (VT.SimpleTy) {
4249 case MVT::i32: return fastEmit_ARMISD_EH_SJLJ_SETJMP_MVT_i32_rr(RetVT, Op0, Op1);
4250 default: return Register();
4251 }
4252}
4253
4254// FastEmit functions for ARMISD::QADD16b.
4255
4256Register fastEmit_ARMISD_QADD16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4257 if (RetVT.SimpleTy != MVT::i32)
4258 return Register();
4259 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4260 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QADD16, RC: &ARM::rGPRRegClass, Op0, Op1);
4261 }
4262 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4263 return fastEmitInst_rr(MachineInstOpcode: ARM::QADD16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4264 }
4265 return Register();
4266}
4267
4268Register fastEmit_ARMISD_QADD16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4269 switch (VT.SimpleTy) {
4270 case MVT::i32: return fastEmit_ARMISD_QADD16b_MVT_i32_rr(RetVT, Op0, Op1);
4271 default: return Register();
4272 }
4273}
4274
4275// FastEmit functions for ARMISD::QADD8b.
4276
4277Register fastEmit_ARMISD_QADD8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4278 if (RetVT.SimpleTy != MVT::i32)
4279 return Register();
4280 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4281 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QADD8, RC: &ARM::rGPRRegClass, Op0, Op1);
4282 }
4283 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4284 return fastEmitInst_rr(MachineInstOpcode: ARM::QADD8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4285 }
4286 return Register();
4287}
4288
4289Register fastEmit_ARMISD_QADD8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4290 switch (VT.SimpleTy) {
4291 case MVT::i32: return fastEmit_ARMISD_QADD8b_MVT_i32_rr(RetVT, Op0, Op1);
4292 default: return Register();
4293 }
4294}
4295
4296// FastEmit functions for ARMISD::QSUB16b.
4297
4298Register fastEmit_ARMISD_QSUB16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4299 if (RetVT.SimpleTy != MVT::i32)
4300 return Register();
4301 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4302 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QSUB16, RC: &ARM::rGPRRegClass, Op0, Op1);
4303 }
4304 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4305 return fastEmitInst_rr(MachineInstOpcode: ARM::QSUB16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4306 }
4307 return Register();
4308}
4309
4310Register fastEmit_ARMISD_QSUB16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4311 switch (VT.SimpleTy) {
4312 case MVT::i32: return fastEmit_ARMISD_QSUB16b_MVT_i32_rr(RetVT, Op0, Op1);
4313 default: return Register();
4314 }
4315}
4316
4317// FastEmit functions for ARMISD::QSUB8b.
4318
4319Register fastEmit_ARMISD_QSUB8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4320 if (RetVT.SimpleTy != MVT::i32)
4321 return Register();
4322 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4323 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QSUB8, RC: &ARM::rGPRRegClass, Op0, Op1);
4324 }
4325 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4326 return fastEmitInst_rr(MachineInstOpcode: ARM::QSUB8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4327 }
4328 return Register();
4329}
4330
4331Register fastEmit_ARMISD_QSUB8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4332 switch (VT.SimpleTy) {
4333 case MVT::i32: return fastEmit_ARMISD_QSUB8b_MVT_i32_rr(RetVT, Op0, Op1);
4334 default: return Register();
4335 }
4336}
4337
4338// FastEmit functions for ARMISD::SMULWB.
4339
4340Register fastEmit_ARMISD_SMULWB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4341 if (RetVT.SimpleTy != MVT::i32)
4342 return Register();
4343 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4344 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SMULWB, RC: &ARM::rGPRRegClass, Op0, Op1);
4345 }
4346 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
4347 return fastEmitInst_rr(MachineInstOpcode: ARM::SMULWB, RC: &ARM::GPRRegClass, Op0, Op1);
4348 }
4349 return Register();
4350}
4351
4352Register fastEmit_ARMISD_SMULWB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4353 switch (VT.SimpleTy) {
4354 case MVT::i32: return fastEmit_ARMISD_SMULWB_MVT_i32_rr(RetVT, Op0, Op1);
4355 default: return Register();
4356 }
4357}
4358
4359// FastEmit functions for ARMISD::SMULWT.
4360
4361Register fastEmit_ARMISD_SMULWT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4362 if (RetVT.SimpleTy != MVT::i32)
4363 return Register();
4364 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4365 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SMULWT, RC: &ARM::rGPRRegClass, Op0, Op1);
4366 }
4367 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
4368 return fastEmitInst_rr(MachineInstOpcode: ARM::SMULWT, RC: &ARM::GPRRegClass, Op0, Op1);
4369 }
4370 return Register();
4371}
4372
4373Register fastEmit_ARMISD_SMULWT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4374 switch (VT.SimpleTy) {
4375 case MVT::i32: return fastEmit_ARMISD_SMULWT_MVT_i32_rr(RetVT, Op0, Op1);
4376 default: return Register();
4377 }
4378}
4379
4380// FastEmit functions for ARMISD::UQADD16b.
4381
4382Register fastEmit_ARMISD_UQADD16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4383 if (RetVT.SimpleTy != MVT::i32)
4384 return Register();
4385 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4386 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQADD16, RC: &ARM::rGPRRegClass, Op0, Op1);
4387 }
4388 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4389 return fastEmitInst_rr(MachineInstOpcode: ARM::UQADD16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4390 }
4391 return Register();
4392}
4393
4394Register fastEmit_ARMISD_UQADD16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4395 switch (VT.SimpleTy) {
4396 case MVT::i32: return fastEmit_ARMISD_UQADD16b_MVT_i32_rr(RetVT, Op0, Op1);
4397 default: return Register();
4398 }
4399}
4400
4401// FastEmit functions for ARMISD::UQADD8b.
4402
4403Register fastEmit_ARMISD_UQADD8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4404 if (RetVT.SimpleTy != MVT::i32)
4405 return Register();
4406 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4407 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQADD8, RC: &ARM::rGPRRegClass, Op0, Op1);
4408 }
4409 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4410 return fastEmitInst_rr(MachineInstOpcode: ARM::UQADD8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4411 }
4412 return Register();
4413}
4414
4415Register fastEmit_ARMISD_UQADD8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4416 switch (VT.SimpleTy) {
4417 case MVT::i32: return fastEmit_ARMISD_UQADD8b_MVT_i32_rr(RetVT, Op0, Op1);
4418 default: return Register();
4419 }
4420}
4421
4422// FastEmit functions for ARMISD::UQSUB16b.
4423
4424Register fastEmit_ARMISD_UQSUB16b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4425 if (RetVT.SimpleTy != MVT::i32)
4426 return Register();
4427 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4428 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQSUB16, RC: &ARM::rGPRRegClass, Op0, Op1);
4429 }
4430 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4431 return fastEmitInst_rr(MachineInstOpcode: ARM::UQSUB16, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4432 }
4433 return Register();
4434}
4435
4436Register fastEmit_ARMISD_UQSUB16b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4437 switch (VT.SimpleTy) {
4438 case MVT::i32: return fastEmit_ARMISD_UQSUB16b_MVT_i32_rr(RetVT, Op0, Op1);
4439 default: return Register();
4440 }
4441}
4442
4443// FastEmit functions for ARMISD::UQSUB8b.
4444
4445Register fastEmit_ARMISD_UQSUB8b_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4446 if (RetVT.SimpleTy != MVT::i32)
4447 return Register();
4448 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4449 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UQSUB8, RC: &ARM::rGPRRegClass, Op0, Op1);
4450 }
4451 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4452 return fastEmitInst_rr(MachineInstOpcode: ARM::UQSUB8, RC: &ARM::GPRnopcRegClass, Op0, Op1);
4453 }
4454 return Register();
4455}
4456
4457Register fastEmit_ARMISD_UQSUB8b_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4458 switch (VT.SimpleTy) {
4459 case MVT::i32: return fastEmit_ARMISD_UQSUB8b_MVT_i32_rr(RetVT, Op0, Op1);
4460 default: return Register();
4461 }
4462}
4463
4464// FastEmit functions for ARMISD::VMLAVs.
4465
4466Register fastEmit_ARMISD_VMLAVs_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4467 if (RetVT.SimpleTy != MVT::i32)
4468 return Register();
4469 if ((Subtarget->hasMVEIntegerOps())) {
4470 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVs8, RC: &ARM::tGPREvenRegClass, Op0, Op1);
4471 }
4472 return Register();
4473}
4474
4475Register fastEmit_ARMISD_VMLAVs_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4476 if (RetVT.SimpleTy != MVT::i32)
4477 return Register();
4478 if ((Subtarget->hasMVEIntegerOps())) {
4479 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVs16, RC: &ARM::tGPREvenRegClass, Op0, Op1);
4480 }
4481 return Register();
4482}
4483
4484Register fastEmit_ARMISD_VMLAVs_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4485 switch (VT.SimpleTy) {
4486 case MVT::v16i8: return fastEmit_ARMISD_VMLAVs_MVT_v16i8_rr(RetVT, Op0, Op1);
4487 case MVT::v8i16: return fastEmit_ARMISD_VMLAVs_MVT_v8i16_rr(RetVT, Op0, Op1);
4488 default: return Register();
4489 }
4490}
4491
4492// FastEmit functions for ARMISD::VMLAVu.
4493
4494Register fastEmit_ARMISD_VMLAVu_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4495 if (RetVT.SimpleTy != MVT::i32)
4496 return Register();
4497 if ((Subtarget->hasMVEIntegerOps())) {
4498 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVu8, RC: &ARM::tGPREvenRegClass, Op0, Op1);
4499 }
4500 return Register();
4501}
4502
4503Register fastEmit_ARMISD_VMLAVu_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4504 if (RetVT.SimpleTy != MVT::i32)
4505 return Register();
4506 if ((Subtarget->hasMVEIntegerOps())) {
4507 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMLADAVu16, RC: &ARM::tGPREvenRegClass, Op0, Op1);
4508 }
4509 return Register();
4510}
4511
4512Register fastEmit_ARMISD_VMLAVu_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4513 switch (VT.SimpleTy) {
4514 case MVT::v16i8: return fastEmit_ARMISD_VMLAVu_MVT_v16i8_rr(RetVT, Op0, Op1);
4515 case MVT::v8i16: return fastEmit_ARMISD_VMLAVu_MVT_v8i16_rr(RetVT, Op0, Op1);
4516 default: return Register();
4517 }
4518}
4519
4520// FastEmit functions for ARMISD::VMOVDRR.
4521
4522Register fastEmit_ARMISD_VMOVDRR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
4523 if (RetVT.SimpleTy != MVT::f64)
4524 return Register();
4525 if ((Subtarget->hasFPRegs())) {
4526 return fastEmitInst_rr(MachineInstOpcode: ARM::VMOVDRR, RC: &ARM::DPRRegClass, Op0, Op1);
4527 }
4528 return Register();
4529}
4530
4531Register fastEmit_ARMISD_VMOVDRR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4532 switch (VT.SimpleTy) {
4533 case MVT::i32: return fastEmit_ARMISD_VMOVDRR_MVT_i32_rr(RetVT, Op0, Op1);
4534 default: return Register();
4535 }
4536}
4537
4538// FastEmit functions for ARMISD::VMULLs.
4539
4540Register fastEmit_ARMISD_VMULLs_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4541 if (RetVT.SimpleTy != MVT::v8i16)
4542 return Register();
4543 if ((Subtarget->hasNEON())) {
4544 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4545 }
4546 return Register();
4547}
4548
4549Register fastEmit_ARMISD_VMULLs_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4550 if (RetVT.SimpleTy != MVT::v4i32)
4551 return Register();
4552 if ((Subtarget->hasNEON())) {
4553 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4554 }
4555 return Register();
4556}
4557
4558Register fastEmit_ARMISD_VMULLs_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4559 if (RetVT.SimpleTy != MVT::v2i64)
4560 return Register();
4561 if ((Subtarget->hasNEON())) {
4562 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4563 }
4564 return Register();
4565}
4566
4567Register fastEmit_ARMISD_VMULLs_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4568 if (RetVT.SimpleTy != MVT::v2i64)
4569 return Register();
4570 if ((Subtarget->hasMVEIntegerOps())) {
4571 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULLBs32, RC: &ARM::MQPRRegClass, Op0, Op1);
4572 }
4573 return Register();
4574}
4575
4576Register fastEmit_ARMISD_VMULLs_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4577 switch (VT.SimpleTy) {
4578 case MVT::v8i8: return fastEmit_ARMISD_VMULLs_MVT_v8i8_rr(RetVT, Op0, Op1);
4579 case MVT::v4i16: return fastEmit_ARMISD_VMULLs_MVT_v4i16_rr(RetVT, Op0, Op1);
4580 case MVT::v2i32: return fastEmit_ARMISD_VMULLs_MVT_v2i32_rr(RetVT, Op0, Op1);
4581 case MVT::v4i32: return fastEmit_ARMISD_VMULLs_MVT_v4i32_rr(RetVT, Op0, Op1);
4582 default: return Register();
4583 }
4584}
4585
4586// FastEmit functions for ARMISD::VMULLu.
4587
4588Register fastEmit_ARMISD_VMULLu_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4589 if (RetVT.SimpleTy != MVT::v8i16)
4590 return Register();
4591 if ((Subtarget->hasNEON())) {
4592 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4593 }
4594 return Register();
4595}
4596
4597Register fastEmit_ARMISD_VMULLu_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4598 if (RetVT.SimpleTy != MVT::v4i32)
4599 return Register();
4600 if ((Subtarget->hasNEON())) {
4601 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4602 }
4603 return Register();
4604}
4605
4606Register fastEmit_ARMISD_VMULLu_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4607 if (RetVT.SimpleTy != MVT::v2i64)
4608 return Register();
4609 if ((Subtarget->hasNEON())) {
4610 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULLuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4611 }
4612 return Register();
4613}
4614
4615Register fastEmit_ARMISD_VMULLu_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4616 if (RetVT.SimpleTy != MVT::v2i64)
4617 return Register();
4618 if ((Subtarget->hasMVEIntegerOps())) {
4619 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULLBu32, RC: &ARM::MQPRRegClass, Op0, Op1);
4620 }
4621 return Register();
4622}
4623
4624Register fastEmit_ARMISD_VMULLu_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4625 switch (VT.SimpleTy) {
4626 case MVT::v8i8: return fastEmit_ARMISD_VMULLu_MVT_v8i8_rr(RetVT, Op0, Op1);
4627 case MVT::v4i16: return fastEmit_ARMISD_VMULLu_MVT_v4i16_rr(RetVT, Op0, Op1);
4628 case MVT::v2i32: return fastEmit_ARMISD_VMULLu_MVT_v2i32_rr(RetVT, Op0, Op1);
4629 case MVT::v4i32: return fastEmit_ARMISD_VMULLu_MVT_v4i32_rr(RetVT, Op0, Op1);
4630 default: return Register();
4631 }
4632}
4633
4634// FastEmit functions for ARMISD::VQDMULH.
4635
4636Register fastEmit_ARMISD_VQDMULH_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4637 if (RetVT.SimpleTy != MVT::v16i8)
4638 return Register();
4639 if ((Subtarget->hasMVEIntegerOps())) {
4640 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQDMULHi8, RC: &ARM::MQPRRegClass, Op0, Op1);
4641 }
4642 return Register();
4643}
4644
4645Register fastEmit_ARMISD_VQDMULH_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4646 if (RetVT.SimpleTy != MVT::v8i16)
4647 return Register();
4648 if ((Subtarget->hasMVEIntegerOps())) {
4649 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQDMULHi16, RC: &ARM::MQPRRegClass, Op0, Op1);
4650 }
4651 return Register();
4652}
4653
4654Register fastEmit_ARMISD_VQDMULH_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4655 if (RetVT.SimpleTy != MVT::v4i32)
4656 return Register();
4657 if ((Subtarget->hasMVEIntegerOps())) {
4658 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQDMULHi32, RC: &ARM::MQPRRegClass, Op0, Op1);
4659 }
4660 return Register();
4661}
4662
4663Register fastEmit_ARMISD_VQDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4664 switch (VT.SimpleTy) {
4665 case MVT::v16i8: return fastEmit_ARMISD_VQDMULH_MVT_v16i8_rr(RetVT, Op0, Op1);
4666 case MVT::v8i16: return fastEmit_ARMISD_VQDMULH_MVT_v8i16_rr(RetVT, Op0, Op1);
4667 case MVT::v4i32: return fastEmit_ARMISD_VQDMULH_MVT_v4i32_rr(RetVT, Op0, Op1);
4668 default: return Register();
4669 }
4670}
4671
4672// FastEmit functions for ARMISD::VSHLs.
4673
4674Register fastEmit_ARMISD_VSHLs_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4675 if (RetVT.SimpleTy != MVT::v8i8)
4676 return Register();
4677 if ((Subtarget->hasNEON())) {
4678 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4679 }
4680 return Register();
4681}
4682
4683Register fastEmit_ARMISD_VSHLs_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4684 if (RetVT.SimpleTy != MVT::v16i8)
4685 return Register();
4686 if ((Subtarget->hasMVEIntegerOps())) {
4687 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecs8, RC: &ARM::MQPRRegClass, Op0, Op1);
4688 }
4689 if ((Subtarget->hasNEON())) {
4690 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4691 }
4692 return Register();
4693}
4694
4695Register fastEmit_ARMISD_VSHLs_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4696 if (RetVT.SimpleTy != MVT::v4i16)
4697 return Register();
4698 if ((Subtarget->hasNEON())) {
4699 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4700 }
4701 return Register();
4702}
4703
4704Register fastEmit_ARMISD_VSHLs_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4705 if (RetVT.SimpleTy != MVT::v8i16)
4706 return Register();
4707 if ((Subtarget->hasMVEIntegerOps())) {
4708 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecs16, RC: &ARM::MQPRRegClass, Op0, Op1);
4709 }
4710 if ((Subtarget->hasNEON())) {
4711 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4712 }
4713 return Register();
4714}
4715
4716Register fastEmit_ARMISD_VSHLs_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4717 if (RetVT.SimpleTy != MVT::v2i32)
4718 return Register();
4719 if ((Subtarget->hasNEON())) {
4720 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4721 }
4722 return Register();
4723}
4724
4725Register fastEmit_ARMISD_VSHLs_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4726 if (RetVT.SimpleTy != MVT::v4i32)
4727 return Register();
4728 if ((Subtarget->hasMVEIntegerOps())) {
4729 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecs32, RC: &ARM::MQPRRegClass, Op0, Op1);
4730 }
4731 if ((Subtarget->hasNEON())) {
4732 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4733 }
4734 return Register();
4735}
4736
4737Register fastEmit_ARMISD_VSHLs_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
4738 if (RetVT.SimpleTy != MVT::v1i64)
4739 return Register();
4740 if ((Subtarget->hasNEON())) {
4741 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
4742 }
4743 return Register();
4744}
4745
4746Register fastEmit_ARMISD_VSHLs_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
4747 if (RetVT.SimpleTy != MVT::v2i64)
4748 return Register();
4749 if ((Subtarget->hasNEON())) {
4750 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4751 }
4752 return Register();
4753}
4754
4755Register fastEmit_ARMISD_VSHLs_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4756 switch (VT.SimpleTy) {
4757 case MVT::v8i8: return fastEmit_ARMISD_VSHLs_MVT_v8i8_rr(RetVT, Op0, Op1);
4758 case MVT::v16i8: return fastEmit_ARMISD_VSHLs_MVT_v16i8_rr(RetVT, Op0, Op1);
4759 case MVT::v4i16: return fastEmit_ARMISD_VSHLs_MVT_v4i16_rr(RetVT, Op0, Op1);
4760 case MVT::v8i16: return fastEmit_ARMISD_VSHLs_MVT_v8i16_rr(RetVT, Op0, Op1);
4761 case MVT::v2i32: return fastEmit_ARMISD_VSHLs_MVT_v2i32_rr(RetVT, Op0, Op1);
4762 case MVT::v4i32: return fastEmit_ARMISD_VSHLs_MVT_v4i32_rr(RetVT, Op0, Op1);
4763 case MVT::v1i64: return fastEmit_ARMISD_VSHLs_MVT_v1i64_rr(RetVT, Op0, Op1);
4764 case MVT::v2i64: return fastEmit_ARMISD_VSHLs_MVT_v2i64_rr(RetVT, Op0, Op1);
4765 default: return Register();
4766 }
4767}
4768
4769// FastEmit functions for ARMISD::VSHLu.
4770
4771Register fastEmit_ARMISD_VSHLu_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4772 if (RetVT.SimpleTy != MVT::v8i8)
4773 return Register();
4774 if ((Subtarget->hasNEON())) {
4775 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4776 }
4777 return Register();
4778}
4779
4780Register fastEmit_ARMISD_VSHLu_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4781 if (RetVT.SimpleTy != MVT::v16i8)
4782 return Register();
4783 if ((Subtarget->hasMVEIntegerOps())) {
4784 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecu8, RC: &ARM::MQPRRegClass, Op0, Op1);
4785 }
4786 if ((Subtarget->hasNEON())) {
4787 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4788 }
4789 return Register();
4790}
4791
4792Register fastEmit_ARMISD_VSHLu_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4793 if (RetVT.SimpleTy != MVT::v4i16)
4794 return Register();
4795 if ((Subtarget->hasNEON())) {
4796 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4797 }
4798 return Register();
4799}
4800
4801Register fastEmit_ARMISD_VSHLu_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4802 if (RetVT.SimpleTy != MVT::v8i16)
4803 return Register();
4804 if ((Subtarget->hasMVEIntegerOps())) {
4805 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecu16, RC: &ARM::MQPRRegClass, Op0, Op1);
4806 }
4807 if ((Subtarget->hasNEON())) {
4808 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4809 }
4810 return Register();
4811}
4812
4813Register fastEmit_ARMISD_VSHLu_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4814 if (RetVT.SimpleTy != MVT::v2i32)
4815 return Register();
4816 if ((Subtarget->hasNEON())) {
4817 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4818 }
4819 return Register();
4820}
4821
4822Register fastEmit_ARMISD_VSHLu_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4823 if (RetVT.SimpleTy != MVT::v4i32)
4824 return Register();
4825 if ((Subtarget->hasMVEIntegerOps())) {
4826 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSHL_by_vecu32, RC: &ARM::MQPRRegClass, Op0, Op1);
4827 }
4828 if ((Subtarget->hasNEON())) {
4829 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4830 }
4831 return Register();
4832}
4833
4834Register fastEmit_ARMISD_VSHLu_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
4835 if (RetVT.SimpleTy != MVT::v1i64)
4836 return Register();
4837 if ((Subtarget->hasNEON())) {
4838 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
4839 }
4840 return Register();
4841}
4842
4843Register fastEmit_ARMISD_VSHLu_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
4844 if (RetVT.SimpleTy != MVT::v2i64)
4845 return Register();
4846 if ((Subtarget->hasNEON())) {
4847 return fastEmitInst_rr(MachineInstOpcode: ARM::VSHLuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
4848 }
4849 return Register();
4850}
4851
4852Register fastEmit_ARMISD_VSHLu_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4853 switch (VT.SimpleTy) {
4854 case MVT::v8i8: return fastEmit_ARMISD_VSHLu_MVT_v8i8_rr(RetVT, Op0, Op1);
4855 case MVT::v16i8: return fastEmit_ARMISD_VSHLu_MVT_v16i8_rr(RetVT, Op0, Op1);
4856 case MVT::v4i16: return fastEmit_ARMISD_VSHLu_MVT_v4i16_rr(RetVT, Op0, Op1);
4857 case MVT::v8i16: return fastEmit_ARMISD_VSHLu_MVT_v8i16_rr(RetVT, Op0, Op1);
4858 case MVT::v2i32: return fastEmit_ARMISD_VSHLu_MVT_v2i32_rr(RetVT, Op0, Op1);
4859 case MVT::v4i32: return fastEmit_ARMISD_VSHLu_MVT_v4i32_rr(RetVT, Op0, Op1);
4860 case MVT::v1i64: return fastEmit_ARMISD_VSHLu_MVT_v1i64_rr(RetVT, Op0, Op1);
4861 case MVT::v2i64: return fastEmit_ARMISD_VSHLu_MVT_v2i64_rr(RetVT, Op0, Op1);
4862 default: return Register();
4863 }
4864}
4865
4866// FastEmit functions for ARMISD::VTBL1.
4867
4868Register fastEmit_ARMISD_VTBL1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4869 if (RetVT.SimpleTy != MVT::v8i8)
4870 return Register();
4871 if ((Subtarget->hasNEON())) {
4872 return fastEmitInst_rr(MachineInstOpcode: ARM::VTBL1, RC: &ARM::DPRRegClass, Op0, Op1);
4873 }
4874 return Register();
4875}
4876
4877Register fastEmit_ARMISD_VTBL1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4878 switch (VT.SimpleTy) {
4879 case MVT::v8i8: return fastEmit_ARMISD_VTBL1_MVT_v8i8_rr(RetVT, Op0, Op1);
4880 default: return Register();
4881 }
4882}
4883
4884// FastEmit functions for ARMISD::VTST.
4885
4886Register fastEmit_ARMISD_VTST_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4887 if (RetVT.SimpleTy != MVT::v8i8)
4888 return Register();
4889 if ((Subtarget->hasNEON())) {
4890 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4891 }
4892 return Register();
4893}
4894
4895Register fastEmit_ARMISD_VTST_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4896 if (RetVT.SimpleTy != MVT::v16i8)
4897 return Register();
4898 if ((Subtarget->hasNEON())) {
4899 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4900 }
4901 return Register();
4902}
4903
4904Register fastEmit_ARMISD_VTST_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4905 if (RetVT.SimpleTy != MVT::v4i16)
4906 return Register();
4907 if ((Subtarget->hasNEON())) {
4908 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4909 }
4910 return Register();
4911}
4912
4913Register fastEmit_ARMISD_VTST_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4914 if (RetVT.SimpleTy != MVT::v8i16)
4915 return Register();
4916 if ((Subtarget->hasNEON())) {
4917 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4918 }
4919 return Register();
4920}
4921
4922Register fastEmit_ARMISD_VTST_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4923 if (RetVT.SimpleTy != MVT::v2i32)
4924 return Register();
4925 if ((Subtarget->hasNEON())) {
4926 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
4927 }
4928 return Register();
4929}
4930
4931Register fastEmit_ARMISD_VTST_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
4932 if (RetVT.SimpleTy != MVT::v4i32)
4933 return Register();
4934 if ((Subtarget->hasNEON())) {
4935 return fastEmitInst_rr(MachineInstOpcode: ARM::VTSTv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
4936 }
4937 return Register();
4938}
4939
4940Register fastEmit_ARMISD_VTST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
4941 switch (VT.SimpleTy) {
4942 case MVT::v8i8: return fastEmit_ARMISD_VTST_MVT_v8i8_rr(RetVT, Op0, Op1);
4943 case MVT::v16i8: return fastEmit_ARMISD_VTST_MVT_v16i8_rr(RetVT, Op0, Op1);
4944 case MVT::v4i16: return fastEmit_ARMISD_VTST_MVT_v4i16_rr(RetVT, Op0, Op1);
4945 case MVT::v8i16: return fastEmit_ARMISD_VTST_MVT_v8i16_rr(RetVT, Op0, Op1);
4946 case MVT::v2i32: return fastEmit_ARMISD_VTST_MVT_v2i32_rr(RetVT, Op0, Op1);
4947 case MVT::v4i32: return fastEmit_ARMISD_VTST_MVT_v4i32_rr(RetVT, Op0, Op1);
4948 default: return Register();
4949 }
4950}
4951
4952// FastEmit functions for ISD::ABDS.
4953
4954Register fastEmit_ISD_ABDS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
4955 if (RetVT.SimpleTy != MVT::v8i8)
4956 return Register();
4957 if ((Subtarget->hasNEON())) {
4958 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
4959 }
4960 return Register();
4961}
4962
4963Register fastEmit_ISD_ABDS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
4964 if (RetVT.SimpleTy != MVT::v16i8)
4965 return Register();
4966 if ((Subtarget->hasMVEIntegerOps())) {
4967 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
4968 }
4969 if ((Subtarget->hasNEON())) {
4970 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
4971 }
4972 return Register();
4973}
4974
4975Register fastEmit_ISD_ABDS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
4976 if (RetVT.SimpleTy != MVT::v4i16)
4977 return Register();
4978 if ((Subtarget->hasNEON())) {
4979 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
4980 }
4981 return Register();
4982}
4983
4984Register fastEmit_ISD_ABDS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
4985 if (RetVT.SimpleTy != MVT::v8i16)
4986 return Register();
4987 if ((Subtarget->hasMVEIntegerOps())) {
4988 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
4989 }
4990 if ((Subtarget->hasNEON())) {
4991 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
4992 }
4993 return Register();
4994}
4995
4996Register fastEmit_ISD_ABDS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
4997 if (RetVT.SimpleTy != MVT::v2i32)
4998 return Register();
4999 if ((Subtarget->hasNEON())) {
5000 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
5001 }
5002 return Register();
5003}
5004
5005Register fastEmit_ISD_ABDS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5006 if (RetVT.SimpleTy != MVT::v4i32)
5007 return Register();
5008 if ((Subtarget->hasMVEIntegerOps())) {
5009 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
5010 }
5011 if ((Subtarget->hasNEON())) {
5012 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
5013 }
5014 return Register();
5015}
5016
5017Register fastEmit_ISD_ABDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5018 switch (VT.SimpleTy) {
5019 case MVT::v8i8: return fastEmit_ISD_ABDS_MVT_v8i8_rr(RetVT, Op0, Op1);
5020 case MVT::v16i8: return fastEmit_ISD_ABDS_MVT_v16i8_rr(RetVT, Op0, Op1);
5021 case MVT::v4i16: return fastEmit_ISD_ABDS_MVT_v4i16_rr(RetVT, Op0, Op1);
5022 case MVT::v8i16: return fastEmit_ISD_ABDS_MVT_v8i16_rr(RetVT, Op0, Op1);
5023 case MVT::v2i32: return fastEmit_ISD_ABDS_MVT_v2i32_rr(RetVT, Op0, Op1);
5024 case MVT::v4i32: return fastEmit_ISD_ABDS_MVT_v4i32_rr(RetVT, Op0, Op1);
5025 default: return Register();
5026 }
5027}
5028
5029// FastEmit functions for ISD::ABDU.
5030
5031Register fastEmit_ISD_ABDU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
5032 if (RetVT.SimpleTy != MVT::v8i8)
5033 return Register();
5034 if ((Subtarget->hasNEON())) {
5035 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
5036 }
5037 return Register();
5038}
5039
5040Register fastEmit_ISD_ABDU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5041 if (RetVT.SimpleTy != MVT::v16i8)
5042 return Register();
5043 if ((Subtarget->hasMVEIntegerOps())) {
5044 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
5045 }
5046 if ((Subtarget->hasNEON())) {
5047 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
5048 }
5049 return Register();
5050}
5051
5052Register fastEmit_ISD_ABDU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
5053 if (RetVT.SimpleTy != MVT::v4i16)
5054 return Register();
5055 if ((Subtarget->hasNEON())) {
5056 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
5057 }
5058 return Register();
5059}
5060
5061Register fastEmit_ISD_ABDU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5062 if (RetVT.SimpleTy != MVT::v8i16)
5063 return Register();
5064 if ((Subtarget->hasMVEIntegerOps())) {
5065 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
5066 }
5067 if ((Subtarget->hasNEON())) {
5068 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
5069 }
5070 return Register();
5071}
5072
5073Register fastEmit_ISD_ABDU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
5074 if (RetVT.SimpleTy != MVT::v2i32)
5075 return Register();
5076 if ((Subtarget->hasNEON())) {
5077 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
5078 }
5079 return Register();
5080}
5081
5082Register fastEmit_ISD_ABDU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5083 if (RetVT.SimpleTy != MVT::v4i32)
5084 return Register();
5085 if ((Subtarget->hasMVEIntegerOps())) {
5086 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VABDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
5087 }
5088 if ((Subtarget->hasNEON())) {
5089 return fastEmitInst_rr(MachineInstOpcode: ARM::VABDuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
5090 }
5091 return Register();
5092}
5093
5094Register fastEmit_ISD_ABDU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5095 switch (VT.SimpleTy) {
5096 case MVT::v8i8: return fastEmit_ISD_ABDU_MVT_v8i8_rr(RetVT, Op0, Op1);
5097 case MVT::v16i8: return fastEmit_ISD_ABDU_MVT_v16i8_rr(RetVT, Op0, Op1);
5098 case MVT::v4i16: return fastEmit_ISD_ABDU_MVT_v4i16_rr(RetVT, Op0, Op1);
5099 case MVT::v8i16: return fastEmit_ISD_ABDU_MVT_v8i16_rr(RetVT, Op0, Op1);
5100 case MVT::v2i32: return fastEmit_ISD_ABDU_MVT_v2i32_rr(RetVT, Op0, Op1);
5101 case MVT::v4i32: return fastEmit_ISD_ABDU_MVT_v4i32_rr(RetVT, Op0, Op1);
5102 default: return Register();
5103 }
5104}
5105
5106// FastEmit functions for ISD::ADD.
5107
5108Register fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5109 if (RetVT.SimpleTy != MVT::i32)
5110 return Register();
5111 if ((Subtarget->isThumb2())) {
5112 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ADDrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
5113 }
5114 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5115 return fastEmitInst_rr(MachineInstOpcode: ARM::tADDrr, RC: &ARM::tGPRRegClass, Op0, Op1);
5116 }
5117 if ((!Subtarget->isThumb())) {
5118 return fastEmitInst_rr(MachineInstOpcode: ARM::ADDrr, RC: &ARM::GPRRegClass, Op0, Op1);
5119 }
5120 return Register();
5121}
5122
5123Register fastEmit_ISD_ADD_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
5124 if (RetVT.SimpleTy != MVT::v8i8)
5125 return Register();
5126 if ((Subtarget->hasNEON())) {
5127 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
5128 }
5129 return Register();
5130}
5131
5132Register fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5133 if (RetVT.SimpleTy != MVT::v16i8)
5134 return Register();
5135 if ((Subtarget->hasMVEIntegerOps())) {
5136 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDi8, RC: &ARM::MQPRRegClass, Op0, Op1);
5137 }
5138 if ((Subtarget->hasNEON())) {
5139 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
5140 }
5141 return Register();
5142}
5143
5144Register fastEmit_ISD_ADD_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
5145 if (RetVT.SimpleTy != MVT::v4i16)
5146 return Register();
5147 if ((Subtarget->hasNEON())) {
5148 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
5149 }
5150 return Register();
5151}
5152
5153Register fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5154 if (RetVT.SimpleTy != MVT::v8i16)
5155 return Register();
5156 if ((Subtarget->hasMVEIntegerOps())) {
5157 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDi16, RC: &ARM::MQPRRegClass, Op0, Op1);
5158 }
5159 if ((Subtarget->hasNEON())) {
5160 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
5161 }
5162 return Register();
5163}
5164
5165Register fastEmit_ISD_ADD_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
5166 if (RetVT.SimpleTy != MVT::v2i32)
5167 return Register();
5168 if ((Subtarget->hasNEON())) {
5169 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
5170 }
5171 return Register();
5172}
5173
5174Register fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5175 if (RetVT.SimpleTy != MVT::v4i32)
5176 return Register();
5177 if ((Subtarget->hasMVEIntegerOps())) {
5178 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDi32, RC: &ARM::MQPRRegClass, Op0, Op1);
5179 }
5180 if ((Subtarget->hasNEON())) {
5181 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
5182 }
5183 return Register();
5184}
5185
5186Register fastEmit_ISD_ADD_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
5187 if (RetVT.SimpleTy != MVT::v1i64)
5188 return Register();
5189 if ((Subtarget->hasNEON())) {
5190 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
5191 }
5192 return Register();
5193}
5194
5195Register fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
5196 if (RetVT.SimpleTy != MVT::v2i64)
5197 return Register();
5198 if ((Subtarget->hasNEON())) {
5199 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
5200 }
5201 return Register();
5202}
5203
5204Register fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5205 switch (VT.SimpleTy) {
5206 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
5207 case MVT::v8i8: return fastEmit_ISD_ADD_MVT_v8i8_rr(RetVT, Op0, Op1);
5208 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
5209 case MVT::v4i16: return fastEmit_ISD_ADD_MVT_v4i16_rr(RetVT, Op0, Op1);
5210 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
5211 case MVT::v2i32: return fastEmit_ISD_ADD_MVT_v2i32_rr(RetVT, Op0, Op1);
5212 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
5213 case MVT::v1i64: return fastEmit_ISD_ADD_MVT_v1i64_rr(RetVT, Op0, Op1);
5214 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
5215 default: return Register();
5216 }
5217}
5218
5219// FastEmit functions for ISD::AND.
5220
5221Register fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
5222 if (RetVT.SimpleTy != MVT::i32)
5223 return Register();
5224 if ((Subtarget->isThumb2())) {
5225 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ANDrr, RC: &ARM::rGPRRegClass, Op0, Op1);
5226 }
5227 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5228 return fastEmitInst_rr(MachineInstOpcode: ARM::tAND, RC: &ARM::tGPRRegClass, Op0, Op1);
5229 }
5230 if ((!Subtarget->isThumb())) {
5231 return fastEmitInst_rr(MachineInstOpcode: ARM::ANDrr, RC: &ARM::GPRRegClass, Op0, Op1);
5232 }
5233 return Register();
5234}
5235
5236Register fastEmit_ISD_AND_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
5237 if (RetVT.SimpleTy != MVT::v8i8)
5238 return Register();
5239 if ((Subtarget->hasNEON())) {
5240 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
5241 }
5242 return Register();
5243}
5244
5245Register fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5246 if (RetVT.SimpleTy != MVT::v16i8)
5247 return Register();
5248 if ((Subtarget->hasMVEIntegerOps())) {
5249 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
5250 }
5251 if ((Subtarget->hasNEON())) {
5252 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
5253 }
5254 return Register();
5255}
5256
5257Register fastEmit_ISD_AND_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
5258 if (RetVT.SimpleTy != MVT::v4i16)
5259 return Register();
5260 if ((Subtarget->hasNEON())) {
5261 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
5262 }
5263 return Register();
5264}
5265
5266Register fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5267 if (RetVT.SimpleTy != MVT::v8i16)
5268 return Register();
5269 if ((Subtarget->hasMVEIntegerOps())) {
5270 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
5271 }
5272 if ((Subtarget->hasNEON())) {
5273 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
5274 }
5275 return Register();
5276}
5277
5278Register fastEmit_ISD_AND_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
5279 if (RetVT.SimpleTy != MVT::v2i32)
5280 return Register();
5281 if ((Subtarget->hasNEON())) {
5282 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
5283 }
5284 return Register();
5285}
5286
5287Register fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5288 if (RetVT.SimpleTy != MVT::v4i32)
5289 return Register();
5290 if ((Subtarget->hasMVEIntegerOps())) {
5291 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
5292 }
5293 if ((Subtarget->hasNEON())) {
5294 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
5295 }
5296 return Register();
5297}
5298
5299Register fastEmit_ISD_AND_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
5300 if (RetVT.SimpleTy != MVT::v1i64)
5301 return Register();
5302 if ((Subtarget->hasNEON())) {
5303 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDd, RC: &ARM::DPRRegClass, Op0, Op1);
5304 }
5305 return Register();
5306}
5307
5308Register fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
5309 if (RetVT.SimpleTy != MVT::v2i64)
5310 return Register();
5311 if ((Subtarget->hasMVEIntegerOps())) {
5312 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VAND, RC: &ARM::MQPRRegClass, Op0, Op1);
5313 }
5314 if ((Subtarget->hasNEON())) {
5315 return fastEmitInst_rr(MachineInstOpcode: ARM::VANDq, RC: &ARM::QPRRegClass, Op0, Op1);
5316 }
5317 return Register();
5318}
5319
5320Register fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5321 switch (VT.SimpleTy) {
5322 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
5323 case MVT::v8i8: return fastEmit_ISD_AND_MVT_v8i8_rr(RetVT, Op0, Op1);
5324 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
5325 case MVT::v4i16: return fastEmit_ISD_AND_MVT_v4i16_rr(RetVT, Op0, Op1);
5326 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
5327 case MVT::v2i32: return fastEmit_ISD_AND_MVT_v2i32_rr(RetVT, Op0, Op1);
5328 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
5329 case MVT::v1i64: return fastEmit_ISD_AND_MVT_v1i64_rr(RetVT, Op0, Op1);
5330 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
5331 default: return Register();
5332 }
5333}
5334
5335// FastEmit functions for ISD::AVGCEILS.
5336
5337Register fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5338 if (RetVT.SimpleTy != MVT::v16i8)
5339 return Register();
5340 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
5341}
5342
5343Register fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5344 if (RetVT.SimpleTy != MVT::v8i16)
5345 return Register();
5346 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
5347}
5348
5349Register fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5350 if (RetVT.SimpleTy != MVT::v4i32)
5351 return Register();
5352 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
5353}
5354
5355Register fastEmit_ISD_AVGCEILS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5356 switch (VT.SimpleTy) {
5357 case MVT::v16i8: return fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(RetVT, Op0, Op1);
5358 case MVT::v8i16: return fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(RetVT, Op0, Op1);
5359 case MVT::v4i32: return fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(RetVT, Op0, Op1);
5360 default: return Register();
5361 }
5362}
5363
5364// FastEmit functions for ISD::AVGCEILU.
5365
5366Register fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5367 if (RetVT.SimpleTy != MVT::v16i8)
5368 return Register();
5369 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
5370}
5371
5372Register fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5373 if (RetVT.SimpleTy != MVT::v8i16)
5374 return Register();
5375 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
5376}
5377
5378Register fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5379 if (RetVT.SimpleTy != MVT::v4i32)
5380 return Register();
5381 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VRHADDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
5382}
5383
5384Register fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5385 switch (VT.SimpleTy) {
5386 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
5387 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
5388 case MVT::v4i32: return fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(RetVT, Op0, Op1);
5389 default: return Register();
5390 }
5391}
5392
5393// FastEmit functions for ISD::AVGFLOORS.
5394
5395Register fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5396 if (RetVT.SimpleTy != MVT::v16i8)
5397 return Register();
5398 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
5399}
5400
5401Register fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5402 if (RetVT.SimpleTy != MVT::v8i16)
5403 return Register();
5404 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
5405}
5406
5407Register fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5408 if (RetVT.SimpleTy != MVT::v4i32)
5409 return Register();
5410 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
5411}
5412
5413Register fastEmit_ISD_AVGFLOORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5414 switch (VT.SimpleTy) {
5415 case MVT::v16i8: return fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(RetVT, Op0, Op1);
5416 case MVT::v8i16: return fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(RetVT, Op0, Op1);
5417 case MVT::v4i32: return fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(RetVT, Op0, Op1);
5418 default: return Register();
5419 }
5420}
5421
5422// FastEmit functions for ISD::AVGFLOORU.
5423
5424Register fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
5425 if (RetVT.SimpleTy != MVT::v16i8)
5426 return Register();
5427 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
5428}
5429
5430Register fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
5431 if (RetVT.SimpleTy != MVT::v8i16)
5432 return Register();
5433 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
5434}
5435
5436Register fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
5437 if (RetVT.SimpleTy != MVT::v4i32)
5438 return Register();
5439 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VHADDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
5440}
5441
5442Register fastEmit_ISD_AVGFLOORU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5443 switch (VT.SimpleTy) {
5444 case MVT::v16i8: return fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(RetVT, Op0, Op1);
5445 case MVT::v8i16: return fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(RetVT, Op0, Op1);
5446 case MVT::v4i32: return fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(RetVT, Op0, Op1);
5447 default: return Register();
5448 }
5449}
5450
5451// FastEmit functions for ISD::FADD.
5452
5453Register fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5454 if (RetVT.SimpleTy != MVT::f16)
5455 return Register();
5456 if ((Subtarget->hasFullFP16())) {
5457 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDH, RC: &ARM::HPRRegClass, Op0, Op1);
5458 }
5459 return Register();
5460}
5461
5462Register fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5463 if (RetVT.SimpleTy != MVT::f32)
5464 return Register();
5465 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
5466 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDS, RC: &ARM::SPRRegClass, Op0, Op1);
5467 }
5468 return Register();
5469}
5470
5471Register fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5472 if (RetVT.SimpleTy != MVT::f64)
5473 return Register();
5474 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
5475 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDD, RC: &ARM::DPRRegClass, Op0, Op1);
5476 }
5477 return Register();
5478}
5479
5480Register fastEmit_ISD_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5481 if (RetVT.SimpleTy != MVT::v4f16)
5482 return Register();
5483 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5484 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDhd, RC: &ARM::DPRRegClass, Op0, Op1);
5485 }
5486 return Register();
5487}
5488
5489Register fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5490 if (RetVT.SimpleTy != MVT::v8f16)
5491 return Register();
5492 if ((Subtarget->hasMVEFloatOps())) {
5493 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5494 }
5495 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5496 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDhq, RC: &ARM::QPRRegClass, Op0, Op1);
5497 }
5498 return Register();
5499}
5500
5501Register fastEmit_ISD_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5502 if (RetVT.SimpleTy != MVT::v2f32)
5503 return Register();
5504 if ((Subtarget->hasNEON())) {
5505 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDfd, RC: &ARM::DPRRegClass, Op0, Op1);
5506 }
5507 return Register();
5508}
5509
5510Register fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5511 if (RetVT.SimpleTy != MVT::v4f32)
5512 return Register();
5513 if ((Subtarget->hasMVEFloatOps())) {
5514 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VADDf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5515 }
5516 if ((Subtarget->hasNEON())) {
5517 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDfq, RC: &ARM::QPRRegClass, Op0, Op1);
5518 }
5519 return Register();
5520}
5521
5522Register fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5523 switch (VT.SimpleTy) {
5524 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
5525 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
5526 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
5527 case MVT::v4f16: return fastEmit_ISD_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
5528 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
5529 case MVT::v2f32: return fastEmit_ISD_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
5530 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
5531 default: return Register();
5532 }
5533}
5534
5535// FastEmit functions for ISD::FDIV.
5536
5537Register fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5538 if (RetVT.SimpleTy != MVT::f16)
5539 return Register();
5540 if ((Subtarget->hasFullFP16())) {
5541 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVH, RC: &ARM::HPRRegClass, Op0, Op1);
5542 }
5543 return Register();
5544}
5545
5546Register fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5547 if (RetVT.SimpleTy != MVT::f32)
5548 return Register();
5549 if ((Subtarget->hasVFP2Base())) {
5550 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVS, RC: &ARM::SPRRegClass, Op0, Op1);
5551 }
5552 return Register();
5553}
5554
5555Register fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5556 if (RetVT.SimpleTy != MVT::f64)
5557 return Register();
5558 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
5559 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVD, RC: &ARM::DPRRegClass, Op0, Op1);
5560 }
5561 return Register();
5562}
5563
5564Register fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5565 switch (VT.SimpleTy) {
5566 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
5567 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
5568 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
5569 default: return Register();
5570 }
5571}
5572
5573// FastEmit functions for ISD::FMAXIMUM.
5574
5575Register fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5576 if (RetVT.SimpleTy != MVT::v4f16)
5577 return Register();
5578 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5579 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXhd, RC: &ARM::DPRRegClass, Op0, Op1);
5580 }
5581 return Register();
5582}
5583
5584Register fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5585 if (RetVT.SimpleTy != MVT::v8f16)
5586 return Register();
5587 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5588 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXhq, RC: &ARM::QPRRegClass, Op0, Op1);
5589 }
5590 return Register();
5591}
5592
5593Register fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5594 if (RetVT.SimpleTy != MVT::v2f32)
5595 return Register();
5596 if ((Subtarget->hasNEON())) {
5597 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXfd, RC: &ARM::DPRRegClass, Op0, Op1);
5598 }
5599 return Register();
5600}
5601
5602Register fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5603 if (RetVT.SimpleTy != MVT::v4f32)
5604 return Register();
5605 if ((Subtarget->hasNEON())) {
5606 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXfq, RC: &ARM::QPRRegClass, Op0, Op1);
5607 }
5608 return Register();
5609}
5610
5611Register fastEmit_ISD_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5612 switch (VT.SimpleTy) {
5613 case MVT::v4f16: return fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
5614 case MVT::v8f16: return fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
5615 case MVT::v2f32: return fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
5616 case MVT::v4f32: return fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
5617 default: return Register();
5618 }
5619}
5620
5621// FastEmit functions for ISD::FMAXNUM.
5622
5623Register fastEmit_ISD_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5624 if (RetVT.SimpleTy != MVT::f16)
5625 return Register();
5626 if ((Subtarget->hasFullFP16())) {
5627 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMH, RC: &ARM::HPRRegClass, Op0, Op1);
5628 }
5629 return Register();
5630}
5631
5632Register fastEmit_ISD_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5633 if (RetVT.SimpleTy != MVT::f32)
5634 return Register();
5635 if ((Subtarget->hasFPARMv8Base())) {
5636 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMS, RC: &ARM::SPRRegClass, Op0, Op1);
5637 }
5638 return Register();
5639}
5640
5641Register fastEmit_ISD_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5642 if (RetVT.SimpleTy != MVT::f64)
5643 return Register();
5644 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
5645 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMD, RC: &ARM::DPRRegClass, Op0, Op1);
5646 }
5647 return Register();
5648}
5649
5650Register fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5651 if (RetVT.SimpleTy != MVT::v4f16)
5652 return Register();
5653 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5654 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNDh, RC: &ARM::DPRRegClass, Op0, Op1);
5655 }
5656 return Register();
5657}
5658
5659Register fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5660 if (RetVT.SimpleTy != MVT::v8f16)
5661 return Register();
5662 if ((Subtarget->hasMVEFloatOps())) {
5663 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXNMf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5664 }
5665 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5666 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNQh, RC: &ARM::QPRRegClass, Op0, Op1);
5667 }
5668 return Register();
5669}
5670
5671Register fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5672 if (RetVT.SimpleTy != MVT::v2f32)
5673 return Register();
5674 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
5675 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNDf, RC: &ARM::DPRRegClass, Op0, Op1);
5676 }
5677 return Register();
5678}
5679
5680Register fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5681 if (RetVT.SimpleTy != MVT::v4f32)
5682 return Register();
5683 if ((Subtarget->hasMVEFloatOps())) {
5684 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXNMf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5685 }
5686 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
5687 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMAXNMNQf, RC: &ARM::QPRRegClass, Op0, Op1);
5688 }
5689 return Register();
5690}
5691
5692Register fastEmit_ISD_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5693 switch (VT.SimpleTy) {
5694 case MVT::f16: return fastEmit_ISD_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
5695 case MVT::f32: return fastEmit_ISD_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
5696 case MVT::f64: return fastEmit_ISD_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
5697 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
5698 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
5699 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
5700 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
5701 default: return Register();
5702 }
5703}
5704
5705// FastEmit functions for ISD::FMINIMUM.
5706
5707Register fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5708 if (RetVT.SimpleTy != MVT::v4f16)
5709 return Register();
5710 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5711 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINhd, RC: &ARM::DPRRegClass, Op0, Op1);
5712 }
5713 return Register();
5714}
5715
5716Register fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5717 if (RetVT.SimpleTy != MVT::v8f16)
5718 return Register();
5719 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5720 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINhq, RC: &ARM::QPRRegClass, Op0, Op1);
5721 }
5722 return Register();
5723}
5724
5725Register fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5726 if (RetVT.SimpleTy != MVT::v2f32)
5727 return Register();
5728 if ((Subtarget->hasNEON())) {
5729 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINfd, RC: &ARM::DPRRegClass, Op0, Op1);
5730 }
5731 return Register();
5732}
5733
5734Register fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5735 if (RetVT.SimpleTy != MVT::v4f32)
5736 return Register();
5737 if ((Subtarget->hasNEON())) {
5738 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINfq, RC: &ARM::QPRRegClass, Op0, Op1);
5739 }
5740 return Register();
5741}
5742
5743Register fastEmit_ISD_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5744 switch (VT.SimpleTy) {
5745 case MVT::v4f16: return fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
5746 case MVT::v8f16: return fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
5747 case MVT::v2f32: return fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
5748 case MVT::v4f32: return fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
5749 default: return Register();
5750 }
5751}
5752
5753// FastEmit functions for ISD::FMINNUM.
5754
5755Register fastEmit_ISD_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5756 if (RetVT.SimpleTy != MVT::f16)
5757 return Register();
5758 if ((Subtarget->hasFullFP16())) {
5759 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMH, RC: &ARM::HPRRegClass, Op0, Op1);
5760 }
5761 return Register();
5762}
5763
5764Register fastEmit_ISD_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5765 if (RetVT.SimpleTy != MVT::f32)
5766 return Register();
5767 if ((Subtarget->hasFPARMv8Base())) {
5768 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMS, RC: &ARM::SPRRegClass, Op0, Op1);
5769 }
5770 return Register();
5771}
5772
5773Register fastEmit_ISD_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5774 if (RetVT.SimpleTy != MVT::f64)
5775 return Register();
5776 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
5777 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMD, RC: &ARM::DPRRegClass, Op0, Op1);
5778 }
5779 return Register();
5780}
5781
5782Register fastEmit_ISD_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5783 if (RetVT.SimpleTy != MVT::v4f16)
5784 return Register();
5785 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5786 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNDh, RC: &ARM::DPRRegClass, Op0, Op1);
5787 }
5788 return Register();
5789}
5790
5791Register fastEmit_ISD_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5792 if (RetVT.SimpleTy != MVT::v8f16)
5793 return Register();
5794 if ((Subtarget->hasMVEFloatOps())) {
5795 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINNMf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5796 }
5797 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5798 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNQh, RC: &ARM::QPRRegClass, Op0, Op1);
5799 }
5800 return Register();
5801}
5802
5803Register fastEmit_ISD_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5804 if (RetVT.SimpleTy != MVT::v2f32)
5805 return Register();
5806 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
5807 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNDf, RC: &ARM::DPRRegClass, Op0, Op1);
5808 }
5809 return Register();
5810}
5811
5812Register fastEmit_ISD_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5813 if (RetVT.SimpleTy != MVT::v4f32)
5814 return Register();
5815 if ((Subtarget->hasMVEFloatOps())) {
5816 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINNMf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5817 }
5818 if ((Subtarget->hasFPARMv8Base()) && (Subtarget->hasNEON())) {
5819 return fastEmitInst_rr(MachineInstOpcode: ARM::NEON_VMINNMNQf, RC: &ARM::QPRRegClass, Op0, Op1);
5820 }
5821 return Register();
5822}
5823
5824Register fastEmit_ISD_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5825 switch (VT.SimpleTy) {
5826 case MVT::f16: return fastEmit_ISD_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
5827 case MVT::f32: return fastEmit_ISD_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
5828 case MVT::f64: return fastEmit_ISD_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
5829 case MVT::v4f16: return fastEmit_ISD_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
5830 case MVT::v8f16: return fastEmit_ISD_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
5831 case MVT::v2f32: return fastEmit_ISD_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
5832 case MVT::v4f32: return fastEmit_ISD_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
5833 default: return Register();
5834 }
5835}
5836
5837// FastEmit functions for ISD::FMUL.
5838
5839Register fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5840 if (RetVT.SimpleTy != MVT::f16)
5841 return Register();
5842 if ((Subtarget->hasFullFP16())) {
5843 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULH, RC: &ARM::HPRRegClass, Op0, Op1);
5844 }
5845 return Register();
5846}
5847
5848Register fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5849 if (RetVT.SimpleTy != MVT::f32)
5850 return Register();
5851 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
5852 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULS, RC: &ARM::SPRRegClass, Op0, Op1);
5853 }
5854 return Register();
5855}
5856
5857Register fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5858 if (RetVT.SimpleTy != MVT::f64)
5859 return Register();
5860 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
5861 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULD, RC: &ARM::DPRRegClass, Op0, Op1);
5862 }
5863 return Register();
5864}
5865
5866Register fastEmit_ISD_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5867 if (RetVT.SimpleTy != MVT::v4f16)
5868 return Register();
5869 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5870 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULhd, RC: &ARM::DPRRegClass, Op0, Op1);
5871 }
5872 return Register();
5873}
5874
5875Register fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5876 if (RetVT.SimpleTy != MVT::v8f16)
5877 return Register();
5878 if ((Subtarget->hasMVEFloatOps())) {
5879 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5880 }
5881 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5882 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULhq, RC: &ARM::QPRRegClass, Op0, Op1);
5883 }
5884 return Register();
5885}
5886
5887Register fastEmit_ISD_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5888 if (RetVT.SimpleTy != MVT::v2f32)
5889 return Register();
5890 if ((Subtarget->hasNEON())) {
5891 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULfd, RC: &ARM::DPRRegClass, Op0, Op1);
5892 }
5893 return Register();
5894}
5895
5896Register fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5897 if (RetVT.SimpleTy != MVT::v4f32)
5898 return Register();
5899 if ((Subtarget->hasMVEFloatOps())) {
5900 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5901 }
5902 if ((Subtarget->hasNEON())) {
5903 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULfq, RC: &ARM::QPRRegClass, Op0, Op1);
5904 }
5905 return Register();
5906}
5907
5908Register fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5909 switch (VT.SimpleTy) {
5910 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
5911 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
5912 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
5913 case MVT::v4f16: return fastEmit_ISD_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
5914 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
5915 case MVT::v2f32: return fastEmit_ISD_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
5916 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
5917 default: return Register();
5918 }
5919}
5920
5921// FastEmit functions for ISD::FSUB.
5922
5923Register fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
5924 if (RetVT.SimpleTy != MVT::f16)
5925 return Register();
5926 if ((Subtarget->hasFullFP16())) {
5927 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBH, RC: &ARM::HPRRegClass, Op0, Op1);
5928 }
5929 return Register();
5930}
5931
5932Register fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
5933 if (RetVT.SimpleTy != MVT::f32)
5934 return Register();
5935 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
5936 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBS, RC: &ARM::SPRRegClass, Op0, Op1);
5937 }
5938 return Register();
5939}
5940
5941Register fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
5942 if (RetVT.SimpleTy != MVT::f64)
5943 return Register();
5944 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
5945 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBD, RC: &ARM::DPRRegClass, Op0, Op1);
5946 }
5947 return Register();
5948}
5949
5950Register fastEmit_ISD_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
5951 if (RetVT.SimpleTy != MVT::v4f16)
5952 return Register();
5953 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5954 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBhd, RC: &ARM::DPRRegClass, Op0, Op1);
5955 }
5956 return Register();
5957}
5958
5959Register fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
5960 if (RetVT.SimpleTy != MVT::v8f16)
5961 return Register();
5962 if ((Subtarget->hasMVEFloatOps())) {
5963 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBf16, RC: &ARM::MQPRRegClass, Op0, Op1);
5964 }
5965 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5966 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBhq, RC: &ARM::QPRRegClass, Op0, Op1);
5967 }
5968 return Register();
5969}
5970
5971Register fastEmit_ISD_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
5972 if (RetVT.SimpleTy != MVT::v2f32)
5973 return Register();
5974 if ((Subtarget->hasNEON())) {
5975 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBfd, RC: &ARM::DPRRegClass, Op0, Op1);
5976 }
5977 return Register();
5978}
5979
5980Register fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
5981 if (RetVT.SimpleTy != MVT::v4f32)
5982 return Register();
5983 if ((Subtarget->hasMVEFloatOps())) {
5984 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBf32, RC: &ARM::MQPRRegClass, Op0, Op1);
5985 }
5986 if ((Subtarget->hasNEON())) {
5987 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBfq, RC: &ARM::QPRRegClass, Op0, Op1);
5988 }
5989 return Register();
5990}
5991
5992Register fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
5993 switch (VT.SimpleTy) {
5994 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
5995 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
5996 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
5997 case MVT::v4f16: return fastEmit_ISD_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
5998 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
5999 case MVT::v2f32: return fastEmit_ISD_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
6000 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
6001 default: return Register();
6002 }
6003}
6004
6005// FastEmit functions for ISD::MUL.
6006
6007Register fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6008 if (RetVT.SimpleTy != MVT::i32)
6009 return Register();
6010 if ((Subtarget->isThumb2())) {
6011 return fastEmitInst_rr(MachineInstOpcode: ARM::t2MUL, RC: &ARM::rGPRRegClass, Op0, Op1);
6012 }
6013 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6014 return fastEmitInst_rr(MachineInstOpcode: ARM::tMUL, RC: &ARM::tGPRRegClass, Op0, Op1);
6015 }
6016 if ((!Subtarget->isThumb()) && (!Subtarget->hasV6Ops()) && (Subtarget->useMulOps())) {
6017 return fastEmitInst_rr(MachineInstOpcode: ARM::MULv5, RC: &ARM::GPRnopcRegClass, Op0, Op1);
6018 }
6019 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
6020 return fastEmitInst_rr(MachineInstOpcode: ARM::MUL, RC: &ARM::GPRnopcRegClass, Op0, Op1);
6021 }
6022 return Register();
6023}
6024
6025Register fastEmit_ISD_MUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6026 if (RetVT.SimpleTy != MVT::v8i8)
6027 return Register();
6028 if ((Subtarget->hasNEON())) {
6029 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6030 }
6031 return Register();
6032}
6033
6034Register fastEmit_ISD_MUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6035 if (RetVT.SimpleTy != MVT::v16i8)
6036 return Register();
6037 if ((Subtarget->hasMVEIntegerOps())) {
6038 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULi8, RC: &ARM::MQPRRegClass, Op0, Op1);
6039 }
6040 if ((Subtarget->hasNEON())) {
6041 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6042 }
6043 return Register();
6044}
6045
6046Register fastEmit_ISD_MUL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6047 if (RetVT.SimpleTy != MVT::v4i16)
6048 return Register();
6049 if ((Subtarget->hasNEON())) {
6050 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6051 }
6052 return Register();
6053}
6054
6055Register fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6056 if (RetVT.SimpleTy != MVT::v8i16)
6057 return Register();
6058 if ((Subtarget->hasMVEIntegerOps())) {
6059 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULi16, RC: &ARM::MQPRRegClass, Op0, Op1);
6060 }
6061 if ((Subtarget->hasNEON())) {
6062 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6063 }
6064 return Register();
6065}
6066
6067Register fastEmit_ISD_MUL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6068 if (RetVT.SimpleTy != MVT::v2i32)
6069 return Register();
6070 if ((Subtarget->hasNEON())) {
6071 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6072 }
6073 return Register();
6074}
6075
6076Register fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6077 if (RetVT.SimpleTy != MVT::v4i32)
6078 return Register();
6079 if ((Subtarget->hasMVEIntegerOps())) {
6080 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULi32, RC: &ARM::MQPRRegClass, Op0, Op1);
6081 }
6082 if ((Subtarget->hasNEON())) {
6083 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6084 }
6085 return Register();
6086}
6087
6088Register fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6089 switch (VT.SimpleTy) {
6090 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op1);
6091 case MVT::v8i8: return fastEmit_ISD_MUL_MVT_v8i8_rr(RetVT, Op0, Op1);
6092 case MVT::v16i8: return fastEmit_ISD_MUL_MVT_v16i8_rr(RetVT, Op0, Op1);
6093 case MVT::v4i16: return fastEmit_ISD_MUL_MVT_v4i16_rr(RetVT, Op0, Op1);
6094 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
6095 case MVT::v2i32: return fastEmit_ISD_MUL_MVT_v2i32_rr(RetVT, Op0, Op1);
6096 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
6097 default: return Register();
6098 }
6099}
6100
6101// FastEmit functions for ISD::MULHS.
6102
6103Register fastEmit_ISD_MULHS_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6104 if (RetVT.SimpleTy != MVT::i32)
6105 return Register();
6106 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
6107 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SMMUL, RC: &ARM::rGPRRegClass, Op0, Op1);
6108 }
6109 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
6110 return fastEmitInst_rr(MachineInstOpcode: ARM::SMMUL, RC: &ARM::GPRRegClass, Op0, Op1);
6111 }
6112 return Register();
6113}
6114
6115Register fastEmit_ISD_MULHS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6116 if (RetVT.SimpleTy != MVT::v16i8)
6117 return Register();
6118 if ((Subtarget->hasMVEIntegerOps())) {
6119 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHs8, RC: &ARM::MQPRRegClass, Op0, Op1);
6120 }
6121 return Register();
6122}
6123
6124Register fastEmit_ISD_MULHS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6125 if (RetVT.SimpleTy != MVT::v8i16)
6126 return Register();
6127 if ((Subtarget->hasMVEIntegerOps())) {
6128 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHs16, RC: &ARM::MQPRRegClass, Op0, Op1);
6129 }
6130 return Register();
6131}
6132
6133Register fastEmit_ISD_MULHS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6134 if (RetVT.SimpleTy != MVT::v4i32)
6135 return Register();
6136 if ((Subtarget->hasMVEIntegerOps())) {
6137 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHs32, RC: &ARM::MQPRRegClass, Op0, Op1);
6138 }
6139 return Register();
6140}
6141
6142Register fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6143 switch (VT.SimpleTy) {
6144 case MVT::i32: return fastEmit_ISD_MULHS_MVT_i32_rr(RetVT, Op0, Op1);
6145 case MVT::v16i8: return fastEmit_ISD_MULHS_MVT_v16i8_rr(RetVT, Op0, Op1);
6146 case MVT::v8i16: return fastEmit_ISD_MULHS_MVT_v8i16_rr(RetVT, Op0, Op1);
6147 case MVT::v4i32: return fastEmit_ISD_MULHS_MVT_v4i32_rr(RetVT, Op0, Op1);
6148 default: return Register();
6149 }
6150}
6151
6152// FastEmit functions for ISD::MULHU.
6153
6154Register fastEmit_ISD_MULHU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6155 if (RetVT.SimpleTy != MVT::v16i8)
6156 return Register();
6157 if ((Subtarget->hasMVEIntegerOps())) {
6158 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHu8, RC: &ARM::MQPRRegClass, Op0, Op1);
6159 }
6160 return Register();
6161}
6162
6163Register fastEmit_ISD_MULHU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6164 if (RetVT.SimpleTy != MVT::v8i16)
6165 return Register();
6166 if ((Subtarget->hasMVEIntegerOps())) {
6167 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHu16, RC: &ARM::MQPRRegClass, Op0, Op1);
6168 }
6169 return Register();
6170}
6171
6172Register fastEmit_ISD_MULHU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6173 if (RetVT.SimpleTy != MVT::v4i32)
6174 return Register();
6175 if ((Subtarget->hasMVEIntegerOps())) {
6176 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMULHu32, RC: &ARM::MQPRRegClass, Op0, Op1);
6177 }
6178 return Register();
6179}
6180
6181Register fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6182 switch (VT.SimpleTy) {
6183 case MVT::v16i8: return fastEmit_ISD_MULHU_MVT_v16i8_rr(RetVT, Op0, Op1);
6184 case MVT::v8i16: return fastEmit_ISD_MULHU_MVT_v8i16_rr(RetVT, Op0, Op1);
6185 case MVT::v4i32: return fastEmit_ISD_MULHU_MVT_v4i32_rr(RetVT, Op0, Op1);
6186 default: return Register();
6187 }
6188}
6189
6190// FastEmit functions for ISD::OR.
6191
6192Register fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6193 if (RetVT.SimpleTy != MVT::i32)
6194 return Register();
6195 if ((Subtarget->isThumb2())) {
6196 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ORRrr, RC: &ARM::rGPRRegClass, Op0, Op1);
6197 }
6198 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6199 return fastEmitInst_rr(MachineInstOpcode: ARM::tORR, RC: &ARM::tGPRRegClass, Op0, Op1);
6200 }
6201 if ((!Subtarget->isThumb())) {
6202 return fastEmitInst_rr(MachineInstOpcode: ARM::ORRrr, RC: &ARM::GPRRegClass, Op0, Op1);
6203 }
6204 return Register();
6205}
6206
6207Register fastEmit_ISD_OR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6208 if (RetVT.SimpleTy != MVT::v8i8)
6209 return Register();
6210 if ((Subtarget->hasNEON())) {
6211 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
6212 }
6213 return Register();
6214}
6215
6216Register fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6217 if (RetVT.SimpleTy != MVT::v16i8)
6218 return Register();
6219 if ((Subtarget->hasMVEIntegerOps())) {
6220 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
6221 }
6222 if ((Subtarget->hasNEON())) {
6223 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
6224 }
6225 return Register();
6226}
6227
6228Register fastEmit_ISD_OR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6229 if (RetVT.SimpleTy != MVT::v4i16)
6230 return Register();
6231 if ((Subtarget->hasNEON())) {
6232 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
6233 }
6234 return Register();
6235}
6236
6237Register fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6238 if (RetVT.SimpleTy != MVT::v8i16)
6239 return Register();
6240 if ((Subtarget->hasMVEIntegerOps())) {
6241 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
6242 }
6243 if ((Subtarget->hasNEON())) {
6244 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
6245 }
6246 return Register();
6247}
6248
6249Register fastEmit_ISD_OR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6250 if (RetVT.SimpleTy != MVT::v2i32)
6251 return Register();
6252 if ((Subtarget->hasNEON())) {
6253 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
6254 }
6255 return Register();
6256}
6257
6258Register fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6259 if (RetVT.SimpleTy != MVT::v4i32)
6260 return Register();
6261 if ((Subtarget->hasMVEIntegerOps())) {
6262 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
6263 }
6264 if ((Subtarget->hasNEON())) {
6265 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
6266 }
6267 return Register();
6268}
6269
6270Register fastEmit_ISD_OR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
6271 if (RetVT.SimpleTy != MVT::v1i64)
6272 return Register();
6273 if ((Subtarget->hasNEON())) {
6274 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRd, RC: &ARM::DPRRegClass, Op0, Op1);
6275 }
6276 return Register();
6277}
6278
6279Register fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6280 if (RetVT.SimpleTy != MVT::v2i64)
6281 return Register();
6282 if ((Subtarget->hasMVEIntegerOps())) {
6283 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VORR, RC: &ARM::MQPRRegClass, Op0, Op1);
6284 }
6285 if ((Subtarget->hasNEON())) {
6286 return fastEmitInst_rr(MachineInstOpcode: ARM::VORRq, RC: &ARM::QPRRegClass, Op0, Op1);
6287 }
6288 return Register();
6289}
6290
6291Register fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6292 switch (VT.SimpleTy) {
6293 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
6294 case MVT::v8i8: return fastEmit_ISD_OR_MVT_v8i8_rr(RetVT, Op0, Op1);
6295 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
6296 case MVT::v4i16: return fastEmit_ISD_OR_MVT_v4i16_rr(RetVT, Op0, Op1);
6297 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
6298 case MVT::v2i32: return fastEmit_ISD_OR_MVT_v2i32_rr(RetVT, Op0, Op1);
6299 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
6300 case MVT::v1i64: return fastEmit_ISD_OR_MVT_v1i64_rr(RetVT, Op0, Op1);
6301 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
6302 default: return Register();
6303 }
6304}
6305
6306// FastEmit functions for ISD::ROTR.
6307
6308Register fastEmit_ISD_ROTR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6309 if (RetVT.SimpleTy != MVT::i32)
6310 return Register();
6311 if ((Subtarget->isThumb2())) {
6312 return fastEmitInst_rr(MachineInstOpcode: ARM::t2RORrr, RC: &ARM::rGPRRegClass, Op0, Op1);
6313 }
6314 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6315 return fastEmitInst_rr(MachineInstOpcode: ARM::tROR, RC: &ARM::tGPRRegClass, Op0, Op1);
6316 }
6317 return Register();
6318}
6319
6320Register fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6321 switch (VT.SimpleTy) {
6322 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_rr(RetVT, Op0, Op1);
6323 default: return Register();
6324 }
6325}
6326
6327// FastEmit functions for ISD::SADDSAT.
6328
6329Register fastEmit_ISD_SADDSAT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6330 if (RetVT.SimpleTy != MVT::i32)
6331 return Register();
6332 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
6333 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QADD, RC: &ARM::rGPRRegClass, Op0, Op1);
6334 }
6335 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
6336 return fastEmitInst_rr(MachineInstOpcode: ARM::QADD, RC: &ARM::GPRnopcRegClass, Op0, Op1);
6337 }
6338 return Register();
6339}
6340
6341Register fastEmit_ISD_SADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6342 if (RetVT.SimpleTy != MVT::v8i8)
6343 return Register();
6344 if ((Subtarget->hasNEON())) {
6345 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6346 }
6347 return Register();
6348}
6349
6350Register fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6351 if (RetVT.SimpleTy != MVT::v16i8)
6352 return Register();
6353 if ((Subtarget->hasMVEIntegerOps())) {
6354 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDs8, RC: &ARM::MQPRRegClass, Op0, Op1);
6355 }
6356 if ((Subtarget->hasNEON())) {
6357 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6358 }
6359 return Register();
6360}
6361
6362Register fastEmit_ISD_SADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6363 if (RetVT.SimpleTy != MVT::v4i16)
6364 return Register();
6365 if ((Subtarget->hasNEON())) {
6366 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6367 }
6368 return Register();
6369}
6370
6371Register fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6372 if (RetVT.SimpleTy != MVT::v8i16)
6373 return Register();
6374 if ((Subtarget->hasMVEIntegerOps())) {
6375 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDs16, RC: &ARM::MQPRRegClass, Op0, Op1);
6376 }
6377 if ((Subtarget->hasNEON())) {
6378 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6379 }
6380 return Register();
6381}
6382
6383Register fastEmit_ISD_SADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6384 if (RetVT.SimpleTy != MVT::v2i32)
6385 return Register();
6386 if ((Subtarget->hasNEON())) {
6387 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6388 }
6389 return Register();
6390}
6391
6392Register fastEmit_ISD_SADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6393 if (RetVT.SimpleTy != MVT::v4i32)
6394 return Register();
6395 if ((Subtarget->hasMVEIntegerOps())) {
6396 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDs32, RC: &ARM::MQPRRegClass, Op0, Op1);
6397 }
6398 if ((Subtarget->hasNEON())) {
6399 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6400 }
6401 return Register();
6402}
6403
6404Register fastEmit_ISD_SADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
6405 if (RetVT.SimpleTy != MVT::v1i64)
6406 return Register();
6407 if ((Subtarget->hasNEON())) {
6408 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
6409 }
6410 return Register();
6411}
6412
6413Register fastEmit_ISD_SADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6414 if (RetVT.SimpleTy != MVT::v2i64)
6415 return Register();
6416 if ((Subtarget->hasNEON())) {
6417 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
6418 }
6419 return Register();
6420}
6421
6422Register fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6423 switch (VT.SimpleTy) {
6424 case MVT::i32: return fastEmit_ISD_SADDSAT_MVT_i32_rr(RetVT, Op0, Op1);
6425 case MVT::v8i8: return fastEmit_ISD_SADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
6426 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
6427 case MVT::v4i16: return fastEmit_ISD_SADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
6428 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
6429 case MVT::v2i32: return fastEmit_ISD_SADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
6430 case MVT::v4i32: return fastEmit_ISD_SADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
6431 case MVT::v1i64: return fastEmit_ISD_SADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
6432 case MVT::v2i64: return fastEmit_ISD_SADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
6433 default: return Register();
6434 }
6435}
6436
6437// FastEmit functions for ISD::SDIV.
6438
6439Register fastEmit_ISD_SDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6440 if (RetVT.SimpleTy != MVT::i32)
6441 return Register();
6442 if ((Subtarget->hasDivideInThumbMode()) && (Subtarget->hasV8MBaselineOps()) && (Subtarget->isThumb())) {
6443 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SDIV, RC: &ARM::rGPRRegClass, Op0, Op1);
6444 }
6445 if ((Subtarget->hasDivideInARMMode()) && (!Subtarget->isThumb())) {
6446 return fastEmitInst_rr(MachineInstOpcode: ARM::SDIV, RC: &ARM::GPRRegClass, Op0, Op1);
6447 }
6448 return Register();
6449}
6450
6451Register fastEmit_ISD_SDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6452 switch (VT.SimpleTy) {
6453 case MVT::i32: return fastEmit_ISD_SDIV_MVT_i32_rr(RetVT, Op0, Op1);
6454 default: return Register();
6455 }
6456}
6457
6458// FastEmit functions for ISD::SHL.
6459
6460Register fastEmit_ISD_SHL_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6461 if (RetVT.SimpleTy != MVT::i32)
6462 return Register();
6463 if ((Subtarget->isThumb2())) {
6464 return fastEmitInst_rr(MachineInstOpcode: ARM::t2LSLrr, RC: &ARM::rGPRRegClass, Op0, Op1);
6465 }
6466 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6467 return fastEmitInst_rr(MachineInstOpcode: ARM::tLSLrr, RC: &ARM::tGPRRegClass, Op0, Op1);
6468 }
6469 return Register();
6470}
6471
6472Register fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6473 switch (VT.SimpleTy) {
6474 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_rr(RetVT, Op0, Op1);
6475 default: return Register();
6476 }
6477}
6478
6479// FastEmit functions for ISD::SMAX.
6480
6481Register fastEmit_ISD_SMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6482 if (RetVT.SimpleTy != MVT::v8i8)
6483 return Register();
6484 if ((Subtarget->hasNEON())) {
6485 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6486 }
6487 return Register();
6488}
6489
6490Register fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6491 if (RetVT.SimpleTy != MVT::v16i8)
6492 return Register();
6493 if ((Subtarget->hasMVEIntegerOps())) {
6494 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXs8, RC: &ARM::MQPRRegClass, Op0, Op1);
6495 }
6496 if ((Subtarget->hasNEON())) {
6497 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6498 }
6499 return Register();
6500}
6501
6502Register fastEmit_ISD_SMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6503 if (RetVT.SimpleTy != MVT::v4i16)
6504 return Register();
6505 if ((Subtarget->hasNEON())) {
6506 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6507 }
6508 return Register();
6509}
6510
6511Register fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6512 if (RetVT.SimpleTy != MVT::v8i16)
6513 return Register();
6514 if ((Subtarget->hasMVEIntegerOps())) {
6515 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXs16, RC: &ARM::MQPRRegClass, Op0, Op1);
6516 }
6517 if ((Subtarget->hasNEON())) {
6518 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6519 }
6520 return Register();
6521}
6522
6523Register fastEmit_ISD_SMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6524 if (RetVT.SimpleTy != MVT::v2i32)
6525 return Register();
6526 if ((Subtarget->hasNEON())) {
6527 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6528 }
6529 return Register();
6530}
6531
6532Register fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6533 if (RetVT.SimpleTy != MVT::v4i32)
6534 return Register();
6535 if ((Subtarget->hasMVEIntegerOps())) {
6536 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXs32, RC: &ARM::MQPRRegClass, Op0, Op1);
6537 }
6538 if ((Subtarget->hasNEON())) {
6539 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6540 }
6541 return Register();
6542}
6543
6544Register fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6545 switch (VT.SimpleTy) {
6546 case MVT::v8i8: return fastEmit_ISD_SMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
6547 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
6548 case MVT::v4i16: return fastEmit_ISD_SMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
6549 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
6550 case MVT::v2i32: return fastEmit_ISD_SMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
6551 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
6552 default: return Register();
6553 }
6554}
6555
6556// FastEmit functions for ISD::SMIN.
6557
6558Register fastEmit_ISD_SMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6559 if (RetVT.SimpleTy != MVT::v8i8)
6560 return Register();
6561 if ((Subtarget->hasNEON())) {
6562 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6563 }
6564 return Register();
6565}
6566
6567Register fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6568 if (RetVT.SimpleTy != MVT::v16i8)
6569 return Register();
6570 if ((Subtarget->hasMVEIntegerOps())) {
6571 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINs8, RC: &ARM::MQPRRegClass, Op0, Op1);
6572 }
6573 if ((Subtarget->hasNEON())) {
6574 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6575 }
6576 return Register();
6577}
6578
6579Register fastEmit_ISD_SMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6580 if (RetVT.SimpleTy != MVT::v4i16)
6581 return Register();
6582 if ((Subtarget->hasNEON())) {
6583 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6584 }
6585 return Register();
6586}
6587
6588Register fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6589 if (RetVT.SimpleTy != MVT::v8i16)
6590 return Register();
6591 if ((Subtarget->hasMVEIntegerOps())) {
6592 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINs16, RC: &ARM::MQPRRegClass, Op0, Op1);
6593 }
6594 if ((Subtarget->hasNEON())) {
6595 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6596 }
6597 return Register();
6598}
6599
6600Register fastEmit_ISD_SMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6601 if (RetVT.SimpleTy != MVT::v2i32)
6602 return Register();
6603 if ((Subtarget->hasNEON())) {
6604 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6605 }
6606 return Register();
6607}
6608
6609Register fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6610 if (RetVT.SimpleTy != MVT::v4i32)
6611 return Register();
6612 if ((Subtarget->hasMVEIntegerOps())) {
6613 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINs32, RC: &ARM::MQPRRegClass, Op0, Op1);
6614 }
6615 if ((Subtarget->hasNEON())) {
6616 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6617 }
6618 return Register();
6619}
6620
6621Register fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6622 switch (VT.SimpleTy) {
6623 case MVT::v8i8: return fastEmit_ISD_SMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
6624 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
6625 case MVT::v4i16: return fastEmit_ISD_SMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
6626 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
6627 case MVT::v2i32: return fastEmit_ISD_SMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
6628 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
6629 default: return Register();
6630 }
6631}
6632
6633// FastEmit functions for ISD::SRA.
6634
6635Register fastEmit_ISD_SRA_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6636 if (RetVT.SimpleTy != MVT::i32)
6637 return Register();
6638 if ((Subtarget->isThumb2())) {
6639 return fastEmitInst_rr(MachineInstOpcode: ARM::t2ASRrr, RC: &ARM::rGPRRegClass, Op0, Op1);
6640 }
6641 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6642 return fastEmitInst_rr(MachineInstOpcode: ARM::tASRrr, RC: &ARM::tGPRRegClass, Op0, Op1);
6643 }
6644 return Register();
6645}
6646
6647Register fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6648 switch (VT.SimpleTy) {
6649 case MVT::i32: return fastEmit_ISD_SRA_MVT_i32_rr(RetVT, Op0, Op1);
6650 default: return Register();
6651 }
6652}
6653
6654// FastEmit functions for ISD::SRL.
6655
6656Register fastEmit_ISD_SRL_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6657 if (RetVT.SimpleTy != MVT::i32)
6658 return Register();
6659 if ((Subtarget->isThumb2())) {
6660 return fastEmitInst_rr(MachineInstOpcode: ARM::t2LSRrr, RC: &ARM::rGPRRegClass, Op0, Op1);
6661 }
6662 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6663 return fastEmitInst_rr(MachineInstOpcode: ARM::tLSRrr, RC: &ARM::tGPRRegClass, Op0, Op1);
6664 }
6665 return Register();
6666}
6667
6668Register fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6669 switch (VT.SimpleTy) {
6670 case MVT::i32: return fastEmit_ISD_SRL_MVT_i32_rr(RetVT, Op0, Op1);
6671 default: return Register();
6672 }
6673}
6674
6675// FastEmit functions for ISD::SSUBSAT.
6676
6677Register fastEmit_ISD_SSUBSAT_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
6678 if (RetVT.SimpleTy != MVT::i32)
6679 return Register();
6680 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
6681 return fastEmitInst_rr(MachineInstOpcode: ARM::t2QSUB, RC: &ARM::rGPRRegClass, Op0, Op1);
6682 }
6683 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
6684 return fastEmitInst_rr(MachineInstOpcode: ARM::QSUB, RC: &ARM::GPRnopcRegClass, Op0, Op1);
6685 }
6686 return Register();
6687}
6688
6689Register fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6690 if (RetVT.SimpleTy != MVT::v8i8)
6691 return Register();
6692 if ((Subtarget->hasNEON())) {
6693 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
6694 }
6695 return Register();
6696}
6697
6698Register fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6699 if (RetVT.SimpleTy != MVT::v16i8)
6700 return Register();
6701 if ((Subtarget->hasMVEIntegerOps())) {
6702 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBs8, RC: &ARM::MQPRRegClass, Op0, Op1);
6703 }
6704 if ((Subtarget->hasNEON())) {
6705 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
6706 }
6707 return Register();
6708}
6709
6710Register fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6711 if (RetVT.SimpleTy != MVT::v4i16)
6712 return Register();
6713 if ((Subtarget->hasNEON())) {
6714 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
6715 }
6716 return Register();
6717}
6718
6719Register fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6720 if (RetVT.SimpleTy != MVT::v8i16)
6721 return Register();
6722 if ((Subtarget->hasMVEIntegerOps())) {
6723 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBs16, RC: &ARM::MQPRRegClass, Op0, Op1);
6724 }
6725 if ((Subtarget->hasNEON())) {
6726 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
6727 }
6728 return Register();
6729}
6730
6731Register fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6732 if (RetVT.SimpleTy != MVT::v2i32)
6733 return Register();
6734 if ((Subtarget->hasNEON())) {
6735 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
6736 }
6737 return Register();
6738}
6739
6740Register fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6741 if (RetVT.SimpleTy != MVT::v4i32)
6742 return Register();
6743 if ((Subtarget->hasMVEIntegerOps())) {
6744 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBs32, RC: &ARM::MQPRRegClass, Op0, Op1);
6745 }
6746 if ((Subtarget->hasNEON())) {
6747 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
6748 }
6749 return Register();
6750}
6751
6752Register fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
6753 if (RetVT.SimpleTy != MVT::v1i64)
6754 return Register();
6755 if ((Subtarget->hasNEON())) {
6756 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
6757 }
6758 return Register();
6759}
6760
6761Register fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6762 if (RetVT.SimpleTy != MVT::v2i64)
6763 return Register();
6764 if ((Subtarget->hasNEON())) {
6765 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBsv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
6766 }
6767 return Register();
6768}
6769
6770Register fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6771 switch (VT.SimpleTy) {
6772 case MVT::i32: return fastEmit_ISD_SSUBSAT_MVT_i32_rr(RetVT, Op0, Op1);
6773 case MVT::v8i8: return fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
6774 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
6775 case MVT::v4i16: return fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
6776 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
6777 case MVT::v2i32: return fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
6778 case MVT::v4i32: return fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
6779 case MVT::v1i64: return fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
6780 case MVT::v2i64: return fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
6781 default: return Register();
6782 }
6783}
6784
6785// FastEmit functions for ISD::STRICT_FADD.
6786
6787Register fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6788 if (RetVT.SimpleTy != MVT::f16)
6789 return Register();
6790 if ((Subtarget->hasFullFP16())) {
6791 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDH, RC: &ARM::HPRRegClass, Op0, Op1);
6792 }
6793 return Register();
6794}
6795
6796Register fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6797 if (RetVT.SimpleTy != MVT::f32)
6798 return Register();
6799 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
6800 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDS, RC: &ARM::SPRRegClass, Op0, Op1);
6801 }
6802 return Register();
6803}
6804
6805Register fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6806 if (RetVT.SimpleTy != MVT::f64)
6807 return Register();
6808 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
6809 return fastEmitInst_rr(MachineInstOpcode: ARM::VADDD, RC: &ARM::DPRRegClass, Op0, Op1);
6810 }
6811 return Register();
6812}
6813
6814Register fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6815 switch (VT.SimpleTy) {
6816 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1);
6817 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1);
6818 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1);
6819 default: return Register();
6820 }
6821}
6822
6823// FastEmit functions for ISD::STRICT_FDIV.
6824
6825Register fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6826 if (RetVT.SimpleTy != MVT::f16)
6827 return Register();
6828 if ((Subtarget->hasFullFP16())) {
6829 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVH, RC: &ARM::HPRRegClass, Op0, Op1);
6830 }
6831 return Register();
6832}
6833
6834Register fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6835 if (RetVT.SimpleTy != MVT::f32)
6836 return Register();
6837 if ((Subtarget->hasVFP2Base())) {
6838 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVS, RC: &ARM::SPRRegClass, Op0, Op1);
6839 }
6840 return Register();
6841}
6842
6843Register fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6844 if (RetVT.SimpleTy != MVT::f64)
6845 return Register();
6846 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
6847 return fastEmitInst_rr(MachineInstOpcode: ARM::VDIVD, RC: &ARM::DPRRegClass, Op0, Op1);
6848 }
6849 return Register();
6850}
6851
6852Register fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6853 switch (VT.SimpleTy) {
6854 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
6855 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
6856 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
6857 default: return Register();
6858 }
6859}
6860
6861// FastEmit functions for ISD::STRICT_FMAXNUM.
6862
6863Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6864 if (RetVT.SimpleTy != MVT::f16)
6865 return Register();
6866 if ((Subtarget->hasFullFP16())) {
6867 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMH, RC: &ARM::HPRRegClass, Op0, Op1);
6868 }
6869 return Register();
6870}
6871
6872Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6873 if (RetVT.SimpleTy != MVT::f32)
6874 return Register();
6875 if ((Subtarget->hasFPARMv8Base())) {
6876 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMS, RC: &ARM::SPRRegClass, Op0, Op1);
6877 }
6878 return Register();
6879}
6880
6881Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6882 if (RetVT.SimpleTy != MVT::f64)
6883 return Register();
6884 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
6885 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMAXNMD, RC: &ARM::DPRRegClass, Op0, Op1);
6886 }
6887 return Register();
6888}
6889
6890Register fastEmit_ISD_STRICT_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6891 switch (VT.SimpleTy) {
6892 case MVT::f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
6893 case MVT::f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
6894 case MVT::f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
6895 default: return Register();
6896 }
6897}
6898
6899// FastEmit functions for ISD::STRICT_FMINNUM.
6900
6901Register fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6902 if (RetVT.SimpleTy != MVT::f16)
6903 return Register();
6904 if ((Subtarget->hasFullFP16())) {
6905 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMH, RC: &ARM::HPRRegClass, Op0, Op1);
6906 }
6907 return Register();
6908}
6909
6910Register fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6911 if (RetVT.SimpleTy != MVT::f32)
6912 return Register();
6913 if ((Subtarget->hasFPARMv8Base())) {
6914 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMS, RC: &ARM::SPRRegClass, Op0, Op1);
6915 }
6916 return Register();
6917}
6918
6919Register fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6920 if (RetVT.SimpleTy != MVT::f64)
6921 return Register();
6922 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
6923 return fastEmitInst_rr(MachineInstOpcode: ARM::VFP_VMINNMD, RC: &ARM::DPRRegClass, Op0, Op1);
6924 }
6925 return Register();
6926}
6927
6928Register fastEmit_ISD_STRICT_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6929 switch (VT.SimpleTy) {
6930 case MVT::f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
6931 case MVT::f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
6932 case MVT::f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
6933 default: return Register();
6934 }
6935}
6936
6937// FastEmit functions for ISD::STRICT_FMUL.
6938
6939Register fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6940 if (RetVT.SimpleTy != MVT::f16)
6941 return Register();
6942 if ((Subtarget->hasFullFP16())) {
6943 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULH, RC: &ARM::HPRRegClass, Op0, Op1);
6944 }
6945 return Register();
6946}
6947
6948Register fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6949 if (RetVT.SimpleTy != MVT::f32)
6950 return Register();
6951 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
6952 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULS, RC: &ARM::SPRRegClass, Op0, Op1);
6953 }
6954 return Register();
6955}
6956
6957Register fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6958 if (RetVT.SimpleTy != MVT::f64)
6959 return Register();
6960 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
6961 return fastEmitInst_rr(MachineInstOpcode: ARM::VMULD, RC: &ARM::DPRRegClass, Op0, Op1);
6962 }
6963 return Register();
6964}
6965
6966Register fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6967 switch (VT.SimpleTy) {
6968 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
6969 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
6970 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
6971 default: return Register();
6972 }
6973}
6974
6975// FastEmit functions for ISD::STRICT_FSUB.
6976
6977Register fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
6978 if (RetVT.SimpleTy != MVT::f16)
6979 return Register();
6980 if ((Subtarget->hasFullFP16())) {
6981 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBH, RC: &ARM::HPRRegClass, Op0, Op1);
6982 }
6983 return Register();
6984}
6985
6986Register fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
6987 if (RetVT.SimpleTy != MVT::f32)
6988 return Register();
6989 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
6990 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBS, RC: &ARM::SPRRegClass, Op0, Op1);
6991 }
6992 return Register();
6993}
6994
6995Register fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
6996 if (RetVT.SimpleTy != MVT::f64)
6997 return Register();
6998 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
6999 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBD, RC: &ARM::DPRRegClass, Op0, Op1);
7000 }
7001 return Register();
7002}
7003
7004Register fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7005 switch (VT.SimpleTy) {
7006 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
7007 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
7008 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
7009 default: return Register();
7010 }
7011}
7012
7013// FastEmit functions for ISD::SUB.
7014
7015Register fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
7016 if (RetVT.SimpleTy != MVT::i32)
7017 return Register();
7018 if ((Subtarget->isThumb2())) {
7019 return fastEmitInst_rr(MachineInstOpcode: ARM::t2SUBrr, RC: &ARM::GPRnopcRegClass, Op0, Op1);
7020 }
7021 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
7022 return fastEmitInst_rr(MachineInstOpcode: ARM::tSUBrr, RC: &ARM::tGPRRegClass, Op0, Op1);
7023 }
7024 if ((!Subtarget->isThumb())) {
7025 return fastEmitInst_rr(MachineInstOpcode: ARM::SUBrr, RC: &ARM::GPRRegClass, Op0, Op1);
7026 }
7027 return Register();
7028}
7029
7030Register fastEmit_ISD_SUB_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7031 if (RetVT.SimpleTy != MVT::v8i8)
7032 return Register();
7033 if ((Subtarget->hasNEON())) {
7034 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
7035 }
7036 return Register();
7037}
7038
7039Register fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7040 if (RetVT.SimpleTy != MVT::v16i8)
7041 return Register();
7042 if ((Subtarget->hasMVEIntegerOps())) {
7043 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBi8, RC: &ARM::MQPRRegClass, Op0, Op1);
7044 }
7045 if ((Subtarget->hasNEON())) {
7046 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
7047 }
7048 return Register();
7049}
7050
7051Register fastEmit_ISD_SUB_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7052 if (RetVT.SimpleTy != MVT::v4i16)
7053 return Register();
7054 if ((Subtarget->hasNEON())) {
7055 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
7056 }
7057 return Register();
7058}
7059
7060Register fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7061 if (RetVT.SimpleTy != MVT::v8i16)
7062 return Register();
7063 if ((Subtarget->hasMVEIntegerOps())) {
7064 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBi16, RC: &ARM::MQPRRegClass, Op0, Op1);
7065 }
7066 if ((Subtarget->hasNEON())) {
7067 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
7068 }
7069 return Register();
7070}
7071
7072Register fastEmit_ISD_SUB_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7073 if (RetVT.SimpleTy != MVT::v2i32)
7074 return Register();
7075 if ((Subtarget->hasNEON())) {
7076 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
7077 }
7078 return Register();
7079}
7080
7081Register fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7082 if (RetVT.SimpleTy != MVT::v4i32)
7083 return Register();
7084 if ((Subtarget->hasMVEIntegerOps())) {
7085 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VSUBi32, RC: &ARM::MQPRRegClass, Op0, Op1);
7086 }
7087 if ((Subtarget->hasNEON())) {
7088 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
7089 }
7090 return Register();
7091}
7092
7093Register fastEmit_ISD_SUB_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7094 if (RetVT.SimpleTy != MVT::v1i64)
7095 return Register();
7096 if ((Subtarget->hasNEON())) {
7097 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
7098 }
7099 return Register();
7100}
7101
7102Register fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7103 if (RetVT.SimpleTy != MVT::v2i64)
7104 return Register();
7105 if ((Subtarget->hasNEON())) {
7106 return fastEmitInst_rr(MachineInstOpcode: ARM::VSUBv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
7107 }
7108 return Register();
7109}
7110
7111Register fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7112 switch (VT.SimpleTy) {
7113 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
7114 case MVT::v8i8: return fastEmit_ISD_SUB_MVT_v8i8_rr(RetVT, Op0, Op1);
7115 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
7116 case MVT::v4i16: return fastEmit_ISD_SUB_MVT_v4i16_rr(RetVT, Op0, Op1);
7117 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
7118 case MVT::v2i32: return fastEmit_ISD_SUB_MVT_v2i32_rr(RetVT, Op0, Op1);
7119 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
7120 case MVT::v1i64: return fastEmit_ISD_SUB_MVT_v1i64_rr(RetVT, Op0, Op1);
7121 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
7122 default: return Register();
7123 }
7124}
7125
7126// FastEmit functions for ISD::UADDSAT.
7127
7128Register fastEmit_ISD_UADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7129 if (RetVT.SimpleTy != MVT::v8i8)
7130 return Register();
7131 if ((Subtarget->hasNEON())) {
7132 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
7133 }
7134 return Register();
7135}
7136
7137Register fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7138 if (RetVT.SimpleTy != MVT::v16i8)
7139 return Register();
7140 if ((Subtarget->hasMVEIntegerOps())) {
7141 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDu8, RC: &ARM::MQPRRegClass, Op0, Op1);
7142 }
7143 if ((Subtarget->hasNEON())) {
7144 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
7145 }
7146 return Register();
7147}
7148
7149Register fastEmit_ISD_UADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7150 if (RetVT.SimpleTy != MVT::v4i16)
7151 return Register();
7152 if ((Subtarget->hasNEON())) {
7153 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
7154 }
7155 return Register();
7156}
7157
7158Register fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7159 if (RetVT.SimpleTy != MVT::v8i16)
7160 return Register();
7161 if ((Subtarget->hasMVEIntegerOps())) {
7162 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDu16, RC: &ARM::MQPRRegClass, Op0, Op1);
7163 }
7164 if ((Subtarget->hasNEON())) {
7165 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
7166 }
7167 return Register();
7168}
7169
7170Register fastEmit_ISD_UADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7171 if (RetVT.SimpleTy != MVT::v2i32)
7172 return Register();
7173 if ((Subtarget->hasNEON())) {
7174 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
7175 }
7176 return Register();
7177}
7178
7179Register fastEmit_ISD_UADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7180 if (RetVT.SimpleTy != MVT::v4i32)
7181 return Register();
7182 if ((Subtarget->hasMVEIntegerOps())) {
7183 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQADDu32, RC: &ARM::MQPRRegClass, Op0, Op1);
7184 }
7185 if ((Subtarget->hasNEON())) {
7186 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
7187 }
7188 return Register();
7189}
7190
7191Register fastEmit_ISD_UADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7192 if (RetVT.SimpleTy != MVT::v1i64)
7193 return Register();
7194 if ((Subtarget->hasNEON())) {
7195 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
7196 }
7197 return Register();
7198}
7199
7200Register fastEmit_ISD_UADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7201 if (RetVT.SimpleTy != MVT::v2i64)
7202 return Register();
7203 if ((Subtarget->hasNEON())) {
7204 return fastEmitInst_rr(MachineInstOpcode: ARM::VQADDuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
7205 }
7206 return Register();
7207}
7208
7209Register fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7210 switch (VT.SimpleTy) {
7211 case MVT::v8i8: return fastEmit_ISD_UADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
7212 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
7213 case MVT::v4i16: return fastEmit_ISD_UADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
7214 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
7215 case MVT::v2i32: return fastEmit_ISD_UADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
7216 case MVT::v4i32: return fastEmit_ISD_UADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
7217 case MVT::v1i64: return fastEmit_ISD_UADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
7218 case MVT::v2i64: return fastEmit_ISD_UADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
7219 default: return Register();
7220 }
7221}
7222
7223// FastEmit functions for ISD::UDIV.
7224
7225Register fastEmit_ISD_UDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
7226 if (RetVT.SimpleTy != MVT::i32)
7227 return Register();
7228 if ((Subtarget->hasDivideInThumbMode()) && (Subtarget->hasV8MBaselineOps()) && (Subtarget->isThumb())) {
7229 return fastEmitInst_rr(MachineInstOpcode: ARM::t2UDIV, RC: &ARM::rGPRRegClass, Op0, Op1);
7230 }
7231 if ((Subtarget->hasDivideInARMMode()) && (!Subtarget->isThumb())) {
7232 return fastEmitInst_rr(MachineInstOpcode: ARM::UDIV, RC: &ARM::GPRRegClass, Op0, Op1);
7233 }
7234 return Register();
7235}
7236
7237Register fastEmit_ISD_UDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7238 switch (VT.SimpleTy) {
7239 case MVT::i32: return fastEmit_ISD_UDIV_MVT_i32_rr(RetVT, Op0, Op1);
7240 default: return Register();
7241 }
7242}
7243
7244// FastEmit functions for ISD::UMAX.
7245
7246Register fastEmit_ISD_UMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7247 if (RetVT.SimpleTy != MVT::v8i8)
7248 return Register();
7249 if ((Subtarget->hasNEON())) {
7250 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
7251 }
7252 return Register();
7253}
7254
7255Register fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7256 if (RetVT.SimpleTy != MVT::v16i8)
7257 return Register();
7258 if ((Subtarget->hasMVEIntegerOps())) {
7259 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXu8, RC: &ARM::MQPRRegClass, Op0, Op1);
7260 }
7261 if ((Subtarget->hasNEON())) {
7262 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
7263 }
7264 return Register();
7265}
7266
7267Register fastEmit_ISD_UMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7268 if (RetVT.SimpleTy != MVT::v4i16)
7269 return Register();
7270 if ((Subtarget->hasNEON())) {
7271 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
7272 }
7273 return Register();
7274}
7275
7276Register fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7277 if (RetVT.SimpleTy != MVT::v8i16)
7278 return Register();
7279 if ((Subtarget->hasMVEIntegerOps())) {
7280 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXu16, RC: &ARM::MQPRRegClass, Op0, Op1);
7281 }
7282 if ((Subtarget->hasNEON())) {
7283 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
7284 }
7285 return Register();
7286}
7287
7288Register fastEmit_ISD_UMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7289 if (RetVT.SimpleTy != MVT::v2i32)
7290 return Register();
7291 if ((Subtarget->hasNEON())) {
7292 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
7293 }
7294 return Register();
7295}
7296
7297Register fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7298 if (RetVT.SimpleTy != MVT::v4i32)
7299 return Register();
7300 if ((Subtarget->hasMVEIntegerOps())) {
7301 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMAXu32, RC: &ARM::MQPRRegClass, Op0, Op1);
7302 }
7303 if ((Subtarget->hasNEON())) {
7304 return fastEmitInst_rr(MachineInstOpcode: ARM::VMAXuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
7305 }
7306 return Register();
7307}
7308
7309Register fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7310 switch (VT.SimpleTy) {
7311 case MVT::v8i8: return fastEmit_ISD_UMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
7312 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
7313 case MVT::v4i16: return fastEmit_ISD_UMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
7314 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
7315 case MVT::v2i32: return fastEmit_ISD_UMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
7316 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
7317 default: return Register();
7318 }
7319}
7320
7321// FastEmit functions for ISD::UMIN.
7322
7323Register fastEmit_ISD_UMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7324 if (RetVT.SimpleTy != MVT::v8i8)
7325 return Register();
7326 if ((Subtarget->hasNEON())) {
7327 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
7328 }
7329 return Register();
7330}
7331
7332Register fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7333 if (RetVT.SimpleTy != MVT::v16i8)
7334 return Register();
7335 if ((Subtarget->hasMVEIntegerOps())) {
7336 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINu8, RC: &ARM::MQPRRegClass, Op0, Op1);
7337 }
7338 if ((Subtarget->hasNEON())) {
7339 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
7340 }
7341 return Register();
7342}
7343
7344Register fastEmit_ISD_UMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7345 if (RetVT.SimpleTy != MVT::v4i16)
7346 return Register();
7347 if ((Subtarget->hasNEON())) {
7348 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
7349 }
7350 return Register();
7351}
7352
7353Register fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7354 if (RetVT.SimpleTy != MVT::v8i16)
7355 return Register();
7356 if ((Subtarget->hasMVEIntegerOps())) {
7357 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINu16, RC: &ARM::MQPRRegClass, Op0, Op1);
7358 }
7359 if ((Subtarget->hasNEON())) {
7360 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
7361 }
7362 return Register();
7363}
7364
7365Register fastEmit_ISD_UMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7366 if (RetVT.SimpleTy != MVT::v2i32)
7367 return Register();
7368 if ((Subtarget->hasNEON())) {
7369 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
7370 }
7371 return Register();
7372}
7373
7374Register fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7375 if (RetVT.SimpleTy != MVT::v4i32)
7376 return Register();
7377 if ((Subtarget->hasMVEIntegerOps())) {
7378 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VMINu32, RC: &ARM::MQPRRegClass, Op0, Op1);
7379 }
7380 if ((Subtarget->hasNEON())) {
7381 return fastEmitInst_rr(MachineInstOpcode: ARM::VMINuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
7382 }
7383 return Register();
7384}
7385
7386Register fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7387 switch (VT.SimpleTy) {
7388 case MVT::v8i8: return fastEmit_ISD_UMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
7389 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
7390 case MVT::v4i16: return fastEmit_ISD_UMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
7391 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
7392 case MVT::v2i32: return fastEmit_ISD_UMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
7393 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
7394 default: return Register();
7395 }
7396}
7397
7398// FastEmit functions for ISD::USUBSAT.
7399
7400Register fastEmit_ISD_USUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7401 if (RetVT.SimpleTy != MVT::v8i8)
7402 return Register();
7403 if ((Subtarget->hasNEON())) {
7404 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv8i8, RC: &ARM::DPRRegClass, Op0, Op1);
7405 }
7406 return Register();
7407}
7408
7409Register fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7410 if (RetVT.SimpleTy != MVT::v16i8)
7411 return Register();
7412 if ((Subtarget->hasMVEIntegerOps())) {
7413 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBu8, RC: &ARM::MQPRRegClass, Op0, Op1);
7414 }
7415 if ((Subtarget->hasNEON())) {
7416 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv16i8, RC: &ARM::QPRRegClass, Op0, Op1);
7417 }
7418 return Register();
7419}
7420
7421Register fastEmit_ISD_USUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7422 if (RetVT.SimpleTy != MVT::v4i16)
7423 return Register();
7424 if ((Subtarget->hasNEON())) {
7425 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv4i16, RC: &ARM::DPRRegClass, Op0, Op1);
7426 }
7427 return Register();
7428}
7429
7430Register fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7431 if (RetVT.SimpleTy != MVT::v8i16)
7432 return Register();
7433 if ((Subtarget->hasMVEIntegerOps())) {
7434 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBu16, RC: &ARM::MQPRRegClass, Op0, Op1);
7435 }
7436 if ((Subtarget->hasNEON())) {
7437 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv8i16, RC: &ARM::QPRRegClass, Op0, Op1);
7438 }
7439 return Register();
7440}
7441
7442Register fastEmit_ISD_USUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7443 if (RetVT.SimpleTy != MVT::v2i32)
7444 return Register();
7445 if ((Subtarget->hasNEON())) {
7446 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv2i32, RC: &ARM::DPRRegClass, Op0, Op1);
7447 }
7448 return Register();
7449}
7450
7451Register fastEmit_ISD_USUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7452 if (RetVT.SimpleTy != MVT::v4i32)
7453 return Register();
7454 if ((Subtarget->hasMVEIntegerOps())) {
7455 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VQSUBu32, RC: &ARM::MQPRRegClass, Op0, Op1);
7456 }
7457 if ((Subtarget->hasNEON())) {
7458 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv4i32, RC: &ARM::QPRRegClass, Op0, Op1);
7459 }
7460 return Register();
7461}
7462
7463Register fastEmit_ISD_USUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7464 if (RetVT.SimpleTy != MVT::v1i64)
7465 return Register();
7466 if ((Subtarget->hasNEON())) {
7467 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv1i64, RC: &ARM::DPRRegClass, Op0, Op1);
7468 }
7469 return Register();
7470}
7471
7472Register fastEmit_ISD_USUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7473 if (RetVT.SimpleTy != MVT::v2i64)
7474 return Register();
7475 if ((Subtarget->hasNEON())) {
7476 return fastEmitInst_rr(MachineInstOpcode: ARM::VQSUBuv2i64, RC: &ARM::QPRRegClass, Op0, Op1);
7477 }
7478 return Register();
7479}
7480
7481Register fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7482 switch (VT.SimpleTy) {
7483 case MVT::v8i8: return fastEmit_ISD_USUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
7484 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
7485 case MVT::v4i16: return fastEmit_ISD_USUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
7486 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
7487 case MVT::v2i32: return fastEmit_ISD_USUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
7488 case MVT::v4i32: return fastEmit_ISD_USUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
7489 case MVT::v1i64: return fastEmit_ISD_USUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
7490 case MVT::v2i64: return fastEmit_ISD_USUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
7491 default: return Register();
7492 }
7493}
7494
7495// FastEmit functions for ISD::XOR.
7496
7497Register fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
7498 if (RetVT.SimpleTy != MVT::i32)
7499 return Register();
7500 if ((Subtarget->isThumb2())) {
7501 return fastEmitInst_rr(MachineInstOpcode: ARM::t2EORrr, RC: &ARM::rGPRRegClass, Op0, Op1);
7502 }
7503 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
7504 return fastEmitInst_rr(MachineInstOpcode: ARM::tEOR, RC: &ARM::tGPRRegClass, Op0, Op1);
7505 }
7506 if ((!Subtarget->isThumb())) {
7507 return fastEmitInst_rr(MachineInstOpcode: ARM::EORrr, RC: &ARM::GPRRegClass, Op0, Op1);
7508 }
7509 return Register();
7510}
7511
7512Register fastEmit_ISD_XOR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7513 if (RetVT.SimpleTy != MVT::v8i8)
7514 return Register();
7515 if ((Subtarget->hasNEON())) {
7516 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
7517 }
7518 return Register();
7519}
7520
7521Register fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7522 if (RetVT.SimpleTy != MVT::v16i8)
7523 return Register();
7524 if ((Subtarget->hasMVEIntegerOps())) {
7525 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
7526 }
7527 if ((Subtarget->hasNEON())) {
7528 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
7529 }
7530 return Register();
7531}
7532
7533Register fastEmit_ISD_XOR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7534 if (RetVT.SimpleTy != MVT::v4i16)
7535 return Register();
7536 if ((Subtarget->hasNEON())) {
7537 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
7538 }
7539 return Register();
7540}
7541
7542Register fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7543 if (RetVT.SimpleTy != MVT::v8i16)
7544 return Register();
7545 if ((Subtarget->hasMVEIntegerOps())) {
7546 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
7547 }
7548 if ((Subtarget->hasNEON())) {
7549 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
7550 }
7551 return Register();
7552}
7553
7554Register fastEmit_ISD_XOR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7555 if (RetVT.SimpleTy != MVT::v2i32)
7556 return Register();
7557 if ((Subtarget->hasNEON())) {
7558 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
7559 }
7560 return Register();
7561}
7562
7563Register fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7564 if (RetVT.SimpleTy != MVT::v4i32)
7565 return Register();
7566 if ((Subtarget->hasMVEIntegerOps())) {
7567 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
7568 }
7569 if ((Subtarget->hasNEON())) {
7570 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
7571 }
7572 return Register();
7573}
7574
7575Register fastEmit_ISD_XOR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7576 if (RetVT.SimpleTy != MVT::v1i64)
7577 return Register();
7578 if ((Subtarget->hasNEON())) {
7579 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORd, RC: &ARM::DPRRegClass, Op0, Op1);
7580 }
7581 return Register();
7582}
7583
7584Register fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7585 if (RetVT.SimpleTy != MVT::v2i64)
7586 return Register();
7587 if ((Subtarget->hasMVEIntegerOps())) {
7588 return fastEmitInst_rr(MachineInstOpcode: ARM::MVE_VEOR, RC: &ARM::MQPRRegClass, Op0, Op1);
7589 }
7590 if ((Subtarget->hasNEON())) {
7591 return fastEmitInst_rr(MachineInstOpcode: ARM::VEORq, RC: &ARM::QPRRegClass, Op0, Op1);
7592 }
7593 return Register();
7594}
7595
7596Register fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7597 switch (VT.SimpleTy) {
7598 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
7599 case MVT::v8i8: return fastEmit_ISD_XOR_MVT_v8i8_rr(RetVT, Op0, Op1);
7600 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
7601 case MVT::v4i16: return fastEmit_ISD_XOR_MVT_v4i16_rr(RetVT, Op0, Op1);
7602 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
7603 case MVT::v2i32: return fastEmit_ISD_XOR_MVT_v2i32_rr(RetVT, Op0, Op1);
7604 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
7605 case MVT::v1i64: return fastEmit_ISD_XOR_MVT_v1i64_rr(RetVT, Op0, Op1);
7606 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
7607 default: return Register();
7608 }
7609}
7610
7611// Top-level FastEmit function.
7612
7613Register fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, Register Op1) override {
7614 switch (Opcode) {
7615 case ARMISD::CMP: return fastEmit_ARMISD_CMP_rr(VT, RetVT, Op0, Op1);
7616 case ARMISD::CMPFP: return fastEmit_ARMISD_CMPFP_rr(VT, RetVT, Op0, Op1);
7617 case ARMISD::CMPFPE: return fastEmit_ARMISD_CMPFPE_rr(VT, RetVT, Op0, Op1);
7618 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_rr(VT, RetVT, Op0, Op1);
7619 case ARMISD::EH_SJLJ_LONGJMP: return fastEmit_ARMISD_EH_SJLJ_LONGJMP_rr(VT, RetVT, Op0, Op1);
7620 case ARMISD::EH_SJLJ_SETJMP: return fastEmit_ARMISD_EH_SJLJ_SETJMP_rr(VT, RetVT, Op0, Op1);
7621 case ARMISD::QADD16b: return fastEmit_ARMISD_QADD16b_rr(VT, RetVT, Op0, Op1);
7622 case ARMISD::QADD8b: return fastEmit_ARMISD_QADD8b_rr(VT, RetVT, Op0, Op1);
7623 case ARMISD::QSUB16b: return fastEmit_ARMISD_QSUB16b_rr(VT, RetVT, Op0, Op1);
7624 case ARMISD::QSUB8b: return fastEmit_ARMISD_QSUB8b_rr(VT, RetVT, Op0, Op1);
7625 case ARMISD::SMULWB: return fastEmit_ARMISD_SMULWB_rr(VT, RetVT, Op0, Op1);
7626 case ARMISD::SMULWT: return fastEmit_ARMISD_SMULWT_rr(VT, RetVT, Op0, Op1);
7627 case ARMISD::UQADD16b: return fastEmit_ARMISD_UQADD16b_rr(VT, RetVT, Op0, Op1);
7628 case ARMISD::UQADD8b: return fastEmit_ARMISD_UQADD8b_rr(VT, RetVT, Op0, Op1);
7629 case ARMISD::UQSUB16b: return fastEmit_ARMISD_UQSUB16b_rr(VT, RetVT, Op0, Op1);
7630 case ARMISD::UQSUB8b: return fastEmit_ARMISD_UQSUB8b_rr(VT, RetVT, Op0, Op1);
7631 case ARMISD::VMLAVs: return fastEmit_ARMISD_VMLAVs_rr(VT, RetVT, Op0, Op1);
7632 case ARMISD::VMLAVu: return fastEmit_ARMISD_VMLAVu_rr(VT, RetVT, Op0, Op1);
7633 case ARMISD::VMOVDRR: return fastEmit_ARMISD_VMOVDRR_rr(VT, RetVT, Op0, Op1);
7634 case ARMISD::VMULLs: return fastEmit_ARMISD_VMULLs_rr(VT, RetVT, Op0, Op1);
7635 case ARMISD::VMULLu: return fastEmit_ARMISD_VMULLu_rr(VT, RetVT, Op0, Op1);
7636 case ARMISD::VQDMULH: return fastEmit_ARMISD_VQDMULH_rr(VT, RetVT, Op0, Op1);
7637 case ARMISD::VSHLs: return fastEmit_ARMISD_VSHLs_rr(VT, RetVT, Op0, Op1);
7638 case ARMISD::VSHLu: return fastEmit_ARMISD_VSHLu_rr(VT, RetVT, Op0, Op1);
7639 case ARMISD::VTBL1: return fastEmit_ARMISD_VTBL1_rr(VT, RetVT, Op0, Op1);
7640 case ARMISD::VTST: return fastEmit_ARMISD_VTST_rr(VT, RetVT, Op0, Op1);
7641 case ISD::ABDS: return fastEmit_ISD_ABDS_rr(VT, RetVT, Op0, Op1);
7642 case ISD::ABDU: return fastEmit_ISD_ABDU_rr(VT, RetVT, Op0, Op1);
7643 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
7644 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
7645 case ISD::AVGCEILS: return fastEmit_ISD_AVGCEILS_rr(VT, RetVT, Op0, Op1);
7646 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
7647 case ISD::AVGFLOORS: return fastEmit_ISD_AVGFLOORS_rr(VT, RetVT, Op0, Op1);
7648 case ISD::AVGFLOORU: return fastEmit_ISD_AVGFLOORU_rr(VT, RetVT, Op0, Op1);
7649 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
7650 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
7651 case ISD::FMAXIMUM: return fastEmit_ISD_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
7652 case ISD::FMAXNUM: return fastEmit_ISD_FMAXNUM_rr(VT, RetVT, Op0, Op1);
7653 case ISD::FMINIMUM: return fastEmit_ISD_FMINIMUM_rr(VT, RetVT, Op0, Op1);
7654 case ISD::FMINNUM: return fastEmit_ISD_FMINNUM_rr(VT, RetVT, Op0, Op1);
7655 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
7656 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
7657 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
7658 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
7659 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
7660 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
7661 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
7662 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
7663 case ISD::SDIV: return fastEmit_ISD_SDIV_rr(VT, RetVT, Op0, Op1);
7664 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
7665 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
7666 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
7667 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
7668 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
7669 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
7670 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1);
7671 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1);
7672 case ISD::STRICT_FMAXNUM: return fastEmit_ISD_STRICT_FMAXNUM_rr(VT, RetVT, Op0, Op1);
7673 case ISD::STRICT_FMINNUM: return fastEmit_ISD_STRICT_FMINNUM_rr(VT, RetVT, Op0, Op1);
7674 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1);
7675 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1);
7676 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
7677 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
7678 case ISD::UDIV: return fastEmit_ISD_UDIV_rr(VT, RetVT, Op0, Op1);
7679 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
7680 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
7681 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
7682 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
7683 default: return Register();
7684 }
7685}
7686
7687// FastEmit functions for ARMISD::PIC_ADD.
7688
7689Register fastEmit_ARMISD_PIC_ADD_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7690 if (RetVT.SimpleTy != MVT::i32)
7691 return Register();
7692 if ((Subtarget->isThumb())) {
7693 return fastEmitInst_ri(MachineInstOpcode: ARM::tPICADD, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7694 }
7695 if ((!Subtarget->isThumb())) {
7696 return fastEmitInst_ri(MachineInstOpcode: ARM::PICADD, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7697 }
7698 return Register();
7699}
7700
7701Register fastEmit_ARMISD_PIC_ADD_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7702 switch (VT.SimpleTy) {
7703 case MVT::i32: return fastEmit_ARMISD_PIC_ADD_MVT_i32_ri(RetVT, Op0, imm1);
7704 default: return Register();
7705 }
7706}
7707
7708// FastEmit functions for ARMISD::VDUPLANE.
7709
7710Register fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7711 if (RetVT.SimpleTy != MVT::v8i8)
7712 return Register();
7713 if ((Subtarget->hasNEON())) {
7714 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN8d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7715 }
7716 return Register();
7717}
7718
7719Register fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7720 if (RetVT.SimpleTy != MVT::v4i16)
7721 return Register();
7722 if ((Subtarget->hasNEON())) {
7723 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7724 }
7725 return Register();
7726}
7727
7728Register fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7729 if (RetVT.SimpleTy != MVT::v2i32)
7730 return Register();
7731 if ((Subtarget->hasNEON())) {
7732 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7733 }
7734 return Register();
7735}
7736
7737Register fastEmit_ARMISD_VDUPLANE_MVT_v4f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7738 if (RetVT.SimpleTy != MVT::v4f16)
7739 return Register();
7740 if ((Subtarget->hasNEON())) {
7741 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7742 }
7743 return Register();
7744}
7745
7746Register fastEmit_ARMISD_VDUPLANE_MVT_v4bf16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7747 if (RetVT.SimpleTy != MVT::v4bf16)
7748 return Register();
7749 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
7750 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7751 }
7752 return Register();
7753}
7754
7755Register fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v2f32_ri(Register Op0, uint64_t imm1) {
7756 if ((Subtarget->hasNEON())) {
7757 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32d, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7758 }
7759 return Register();
7760}
7761
7762Register fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v4f32_ri(Register Op0, uint64_t imm1) {
7763 if ((Subtarget->hasNEON())) {
7764 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7765 }
7766 return Register();
7767}
7768
7769Register fastEmit_ARMISD_VDUPLANE_MVT_v2f32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7770switch (RetVT.SimpleTy) {
7771 case MVT::v2f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v2f32_ri(Op0, imm1);
7772 case MVT::v4f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v4f32_ri(Op0, imm1);
7773 default: return Register();
7774}
7775}
7776
7777Register fastEmit_ARMISD_VDUPLANE_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7778 switch (VT.SimpleTy) {
7779 case MVT::v8i8: return fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri(RetVT, Op0, imm1);
7780 case MVT::v4i16: return fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri(RetVT, Op0, imm1);
7781 case MVT::v2i32: return fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri(RetVT, Op0, imm1);
7782 case MVT::v4f16: return fastEmit_ARMISD_VDUPLANE_MVT_v4f16_ri(RetVT, Op0, imm1);
7783 case MVT::v4bf16: return fastEmit_ARMISD_VDUPLANE_MVT_v4bf16_ri(RetVT, Op0, imm1);
7784 case MVT::v2f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_ri(RetVT, Op0, imm1);
7785 default: return Register();
7786 }
7787}
7788
7789// FastEmit functions for ARMISD::VGETLANEs.
7790
7791Register fastEmit_ARMISD_VGETLANEs_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7792 if (RetVT.SimpleTy != MVT::i32)
7793 return Register();
7794 if ((Subtarget->hasNEON())) {
7795 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNs8, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7796 }
7797 return Register();
7798}
7799
7800Register fastEmit_ARMISD_VGETLANEs_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7801 if (RetVT.SimpleTy != MVT::i32)
7802 return Register();
7803 if ((Subtarget->hasMVEIntegerOps())) {
7804 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_s8, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7805 }
7806 return Register();
7807}
7808
7809Register fastEmit_ARMISD_VGETLANEs_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7810 if (RetVT.SimpleTy != MVT::i32)
7811 return Register();
7812 if ((Subtarget->hasNEON())) {
7813 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNs16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7814 }
7815 return Register();
7816}
7817
7818Register fastEmit_ARMISD_VGETLANEs_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7819 if (RetVT.SimpleTy != MVT::i32)
7820 return Register();
7821 if ((Subtarget->hasMVEIntegerOps())) {
7822 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_s16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7823 }
7824 return Register();
7825}
7826
7827Register fastEmit_ARMISD_VGETLANEs_MVT_v8f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7828 if (RetVT.SimpleTy != MVT::i32)
7829 return Register();
7830 if ((Subtarget->hasMVEIntegerOps())) {
7831 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_s16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7832 }
7833 return Register();
7834}
7835
7836Register fastEmit_ARMISD_VGETLANEs_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7837 switch (VT.SimpleTy) {
7838 case MVT::v8i8: return fastEmit_ARMISD_VGETLANEs_MVT_v8i8_ri(RetVT, Op0, imm1);
7839 case MVT::v16i8: return fastEmit_ARMISD_VGETLANEs_MVT_v16i8_ri(RetVT, Op0, imm1);
7840 case MVT::v4i16: return fastEmit_ARMISD_VGETLANEs_MVT_v4i16_ri(RetVT, Op0, imm1);
7841 case MVT::v8i16: return fastEmit_ARMISD_VGETLANEs_MVT_v8i16_ri(RetVT, Op0, imm1);
7842 case MVT::v8f16: return fastEmit_ARMISD_VGETLANEs_MVT_v8f16_ri(RetVT, Op0, imm1);
7843 default: return Register();
7844 }
7845}
7846
7847// FastEmit functions for ARMISD::VGETLANEu.
7848
7849Register fastEmit_ARMISD_VGETLANEu_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7850 if (RetVT.SimpleTy != MVT::i32)
7851 return Register();
7852 if ((Subtarget->hasNEON())) {
7853 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu8, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7854 }
7855 return Register();
7856}
7857
7858Register fastEmit_ARMISD_VGETLANEu_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7859 if (RetVT.SimpleTy != MVT::i32)
7860 return Register();
7861 if ((Subtarget->hasMVEIntegerOps())) {
7862 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_u8, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7863 }
7864 return Register();
7865}
7866
7867Register fastEmit_ARMISD_VGETLANEu_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7868 if (RetVT.SimpleTy != MVT::i32)
7869 return Register();
7870 if ((Subtarget->hasNEON())) {
7871 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7872 }
7873 return Register();
7874}
7875
7876Register fastEmit_ARMISD_VGETLANEu_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7877 if (RetVT.SimpleTy != MVT::i32)
7878 return Register();
7879 if ((Subtarget->hasMVEIntegerOps())) {
7880 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_u16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7881 }
7882 return Register();
7883}
7884
7885Register fastEmit_ARMISD_VGETLANEu_MVT_v4f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7886 if (RetVT.SimpleTy != MVT::i32)
7887 return Register();
7888 if ((Subtarget->hasNEON())) {
7889 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7890 }
7891 return Register();
7892}
7893
7894Register fastEmit_ARMISD_VGETLANEu_MVT_v8f16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7895 if (RetVT.SimpleTy != MVT::i32)
7896 return Register();
7897 if ((Subtarget->hasMVEIntegerOps())) {
7898 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VMOV_from_lane_u16, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
7899 }
7900 return Register();
7901}
7902
7903Register fastEmit_ARMISD_VGETLANEu_MVT_v4bf16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7904 if (RetVT.SimpleTy != MVT::i32)
7905 return Register();
7906 if ((Subtarget->hasNEON())) {
7907 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNu16, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
7908 }
7909 return Register();
7910}
7911
7912Register fastEmit_ARMISD_VGETLANEu_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
7913 switch (VT.SimpleTy) {
7914 case MVT::v8i8: return fastEmit_ARMISD_VGETLANEu_MVT_v8i8_ri(RetVT, Op0, imm1);
7915 case MVT::v16i8: return fastEmit_ARMISD_VGETLANEu_MVT_v16i8_ri(RetVT, Op0, imm1);
7916 case MVT::v4i16: return fastEmit_ARMISD_VGETLANEu_MVT_v4i16_ri(RetVT, Op0, imm1);
7917 case MVT::v8i16: return fastEmit_ARMISD_VGETLANEu_MVT_v8i16_ri(RetVT, Op0, imm1);
7918 case MVT::v4f16: return fastEmit_ARMISD_VGETLANEu_MVT_v4f16_ri(RetVT, Op0, imm1);
7919 case MVT::v8f16: return fastEmit_ARMISD_VGETLANEu_MVT_v8f16_ri(RetVT, Op0, imm1);
7920 case MVT::v4bf16: return fastEmit_ARMISD_VGETLANEu_MVT_v4bf16_ri(RetVT, Op0, imm1);
7921 default: return Register();
7922 }
7923}
7924
7925// FastEmit functions for ARMISD::VQSHLsIMM.
7926
7927Register fastEmit_ARMISD_VQSHLsIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7928 if (RetVT.SimpleTy != MVT::v8i8)
7929 return Register();
7930 if ((Subtarget->hasNEON())) {
7931 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7932 }
7933 return Register();
7934}
7935
7936Register fastEmit_ARMISD_VQSHLsIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7937 if (RetVT.SimpleTy != MVT::v16i8)
7938 return Register();
7939 if ((Subtarget->hasNEON())) {
7940 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7941 }
7942 return Register();
7943}
7944
7945Register fastEmit_ARMISD_VQSHLsIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7946 if (RetVT.SimpleTy != MVT::v4i16)
7947 return Register();
7948 if ((Subtarget->hasNEON())) {
7949 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7950 }
7951 return Register();
7952}
7953
7954Register fastEmit_ARMISD_VQSHLsIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7955 if (RetVT.SimpleTy != MVT::v8i16)
7956 return Register();
7957 if ((Subtarget->hasNEON())) {
7958 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7959 }
7960 return Register();
7961}
7962
7963Register fastEmit_ARMISD_VQSHLsIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7964 if (RetVT.SimpleTy != MVT::v2i32)
7965 return Register();
7966 if ((Subtarget->hasNEON())) {
7967 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7968 }
7969 return Register();
7970}
7971
7972Register fastEmit_ARMISD_VQSHLsIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7973 if (RetVT.SimpleTy != MVT::v4i32)
7974 return Register();
7975 if ((Subtarget->hasNEON())) {
7976 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7977 }
7978 return Register();
7979}
7980
7981Register fastEmit_ARMISD_VQSHLsIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7982 if (RetVT.SimpleTy != MVT::v1i64)
7983 return Register();
7984 if ((Subtarget->hasNEON())) {
7985 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
7986 }
7987 return Register();
7988}
7989
7990Register fastEmit_ARMISD_VQSHLsIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
7991 if (RetVT.SimpleTy != MVT::v2i64)
7992 return Register();
7993 if ((Subtarget->hasNEON())) {
7994 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsiv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
7995 }
7996 return Register();
7997}
7998
7999Register fastEmit_ARMISD_VQSHLsIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8000 switch (VT.SimpleTy) {
8001 case MVT::v8i8: return fastEmit_ARMISD_VQSHLsIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8002 case MVT::v16i8: return fastEmit_ARMISD_VQSHLsIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8003 case MVT::v4i16: return fastEmit_ARMISD_VQSHLsIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8004 case MVT::v8i16: return fastEmit_ARMISD_VQSHLsIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8005 case MVT::v2i32: return fastEmit_ARMISD_VQSHLsIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8006 case MVT::v4i32: return fastEmit_ARMISD_VQSHLsIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8007 case MVT::v1i64: return fastEmit_ARMISD_VQSHLsIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8008 case MVT::v2i64: return fastEmit_ARMISD_VQSHLsIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8009 default: return Register();
8010 }
8011}
8012
8013// FastEmit functions for ARMISD::VQSHLsuIMM.
8014
8015Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8016 if (RetVT.SimpleTy != MVT::v8i8)
8017 return Register();
8018 if ((Subtarget->hasNEON())) {
8019 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8020 }
8021 return Register();
8022}
8023
8024Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8025 if (RetVT.SimpleTy != MVT::v16i8)
8026 return Register();
8027 if ((Subtarget->hasNEON())) {
8028 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8029 }
8030 return Register();
8031}
8032
8033Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8034 if (RetVT.SimpleTy != MVT::v4i16)
8035 return Register();
8036 if ((Subtarget->hasNEON())) {
8037 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8038 }
8039 return Register();
8040}
8041
8042Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8043 if (RetVT.SimpleTy != MVT::v8i16)
8044 return Register();
8045 if ((Subtarget->hasNEON())) {
8046 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8047 }
8048 return Register();
8049}
8050
8051Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8052 if (RetVT.SimpleTy != MVT::v2i32)
8053 return Register();
8054 if ((Subtarget->hasNEON())) {
8055 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8056 }
8057 return Register();
8058}
8059
8060Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8061 if (RetVT.SimpleTy != MVT::v4i32)
8062 return Register();
8063 if ((Subtarget->hasNEON())) {
8064 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8065 }
8066 return Register();
8067}
8068
8069Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8070 if (RetVT.SimpleTy != MVT::v1i64)
8071 return Register();
8072 if ((Subtarget->hasNEON())) {
8073 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8074 }
8075 return Register();
8076}
8077
8078Register fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8079 if (RetVT.SimpleTy != MVT::v2i64)
8080 return Register();
8081 if ((Subtarget->hasNEON())) {
8082 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLsuv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8083 }
8084 return Register();
8085}
8086
8087Register fastEmit_ARMISD_VQSHLsuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8088 switch (VT.SimpleTy) {
8089 case MVT::v8i8: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8090 case MVT::v16i8: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8091 case MVT::v4i16: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8092 case MVT::v8i16: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8093 case MVT::v2i32: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8094 case MVT::v4i32: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8095 case MVT::v1i64: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8096 case MVT::v2i64: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8097 default: return Register();
8098 }
8099}
8100
8101// FastEmit functions for ARMISD::VQSHLuIMM.
8102
8103Register fastEmit_ARMISD_VQSHLuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8104 if (RetVT.SimpleTy != MVT::v8i8)
8105 return Register();
8106 if ((Subtarget->hasNEON())) {
8107 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8108 }
8109 return Register();
8110}
8111
8112Register fastEmit_ARMISD_VQSHLuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8113 if (RetVT.SimpleTy != MVT::v16i8)
8114 return Register();
8115 if ((Subtarget->hasNEON())) {
8116 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8117 }
8118 return Register();
8119}
8120
8121Register fastEmit_ARMISD_VQSHLuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8122 if (RetVT.SimpleTy != MVT::v4i16)
8123 return Register();
8124 if ((Subtarget->hasNEON())) {
8125 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8126 }
8127 return Register();
8128}
8129
8130Register fastEmit_ARMISD_VQSHLuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8131 if (RetVT.SimpleTy != MVT::v8i16)
8132 return Register();
8133 if ((Subtarget->hasNEON())) {
8134 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8135 }
8136 return Register();
8137}
8138
8139Register fastEmit_ARMISD_VQSHLuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8140 if (RetVT.SimpleTy != MVT::v2i32)
8141 return Register();
8142 if ((Subtarget->hasNEON())) {
8143 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8144 }
8145 return Register();
8146}
8147
8148Register fastEmit_ARMISD_VQSHLuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8149 if (RetVT.SimpleTy != MVT::v4i32)
8150 return Register();
8151 if ((Subtarget->hasNEON())) {
8152 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8153 }
8154 return Register();
8155}
8156
8157Register fastEmit_ARMISD_VQSHLuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8158 if (RetVT.SimpleTy != MVT::v1i64)
8159 return Register();
8160 if ((Subtarget->hasNEON())) {
8161 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8162 }
8163 return Register();
8164}
8165
8166Register fastEmit_ARMISD_VQSHLuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8167 if (RetVT.SimpleTy != MVT::v2i64)
8168 return Register();
8169 if ((Subtarget->hasNEON())) {
8170 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHLuiv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8171 }
8172 return Register();
8173}
8174
8175Register fastEmit_ARMISD_VQSHLuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8176 switch (VT.SimpleTy) {
8177 case MVT::v8i8: return fastEmit_ARMISD_VQSHLuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8178 case MVT::v16i8: return fastEmit_ARMISD_VQSHLuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8179 case MVT::v4i16: return fastEmit_ARMISD_VQSHLuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8180 case MVT::v8i16: return fastEmit_ARMISD_VQSHLuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8181 case MVT::v2i32: return fastEmit_ARMISD_VQSHLuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8182 case MVT::v4i32: return fastEmit_ARMISD_VQSHLuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8183 case MVT::v1i64: return fastEmit_ARMISD_VQSHLuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8184 case MVT::v2i64: return fastEmit_ARMISD_VQSHLuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8185 default: return Register();
8186 }
8187}
8188
8189// FastEmit functions for ARMISD::VRSHRsIMM.
8190
8191Register fastEmit_ARMISD_VRSHRsIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8192 if (RetVT.SimpleTy != MVT::v8i8)
8193 return Register();
8194 if ((Subtarget->hasNEON())) {
8195 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8196 }
8197 return Register();
8198}
8199
8200Register fastEmit_ARMISD_VRSHRsIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8201 if (RetVT.SimpleTy != MVT::v16i8)
8202 return Register();
8203 if ((Subtarget->hasNEON())) {
8204 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8205 }
8206 return Register();
8207}
8208
8209Register fastEmit_ARMISD_VRSHRsIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8210 if (RetVT.SimpleTy != MVT::v4i16)
8211 return Register();
8212 if ((Subtarget->hasNEON())) {
8213 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8214 }
8215 return Register();
8216}
8217
8218Register fastEmit_ARMISD_VRSHRsIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8219 if (RetVT.SimpleTy != MVT::v8i16)
8220 return Register();
8221 if ((Subtarget->hasNEON())) {
8222 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8223 }
8224 return Register();
8225}
8226
8227Register fastEmit_ARMISD_VRSHRsIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8228 if (RetVT.SimpleTy != MVT::v2i32)
8229 return Register();
8230 if ((Subtarget->hasNEON())) {
8231 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8232 }
8233 return Register();
8234}
8235
8236Register fastEmit_ARMISD_VRSHRsIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8237 if (RetVT.SimpleTy != MVT::v4i32)
8238 return Register();
8239 if ((Subtarget->hasNEON())) {
8240 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8241 }
8242 return Register();
8243}
8244
8245Register fastEmit_ARMISD_VRSHRsIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8246 if (RetVT.SimpleTy != MVT::v1i64)
8247 return Register();
8248 if ((Subtarget->hasNEON())) {
8249 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8250 }
8251 return Register();
8252}
8253
8254Register fastEmit_ARMISD_VRSHRsIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8255 if (RetVT.SimpleTy != MVT::v2i64)
8256 return Register();
8257 if ((Subtarget->hasNEON())) {
8258 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRsv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8259 }
8260 return Register();
8261}
8262
8263Register fastEmit_ARMISD_VRSHRsIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8264 switch (VT.SimpleTy) {
8265 case MVT::v8i8: return fastEmit_ARMISD_VRSHRsIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8266 case MVT::v16i8: return fastEmit_ARMISD_VRSHRsIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8267 case MVT::v4i16: return fastEmit_ARMISD_VRSHRsIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8268 case MVT::v8i16: return fastEmit_ARMISD_VRSHRsIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8269 case MVT::v2i32: return fastEmit_ARMISD_VRSHRsIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8270 case MVT::v4i32: return fastEmit_ARMISD_VRSHRsIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8271 case MVT::v1i64: return fastEmit_ARMISD_VRSHRsIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8272 case MVT::v2i64: return fastEmit_ARMISD_VRSHRsIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8273 default: return Register();
8274 }
8275}
8276
8277// FastEmit functions for ARMISD::VRSHRuIMM.
8278
8279Register fastEmit_ARMISD_VRSHRuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8280 if (RetVT.SimpleTy != MVT::v8i8)
8281 return Register();
8282 if ((Subtarget->hasNEON())) {
8283 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8284 }
8285 return Register();
8286}
8287
8288Register fastEmit_ARMISD_VRSHRuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8289 if (RetVT.SimpleTy != MVT::v16i8)
8290 return Register();
8291 if ((Subtarget->hasNEON())) {
8292 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8293 }
8294 return Register();
8295}
8296
8297Register fastEmit_ARMISD_VRSHRuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8298 if (RetVT.SimpleTy != MVT::v4i16)
8299 return Register();
8300 if ((Subtarget->hasNEON())) {
8301 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8302 }
8303 return Register();
8304}
8305
8306Register fastEmit_ARMISD_VRSHRuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8307 if (RetVT.SimpleTy != MVT::v8i16)
8308 return Register();
8309 if ((Subtarget->hasNEON())) {
8310 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8311 }
8312 return Register();
8313}
8314
8315Register fastEmit_ARMISD_VRSHRuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8316 if (RetVT.SimpleTy != MVT::v2i32)
8317 return Register();
8318 if ((Subtarget->hasNEON())) {
8319 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8320 }
8321 return Register();
8322}
8323
8324Register fastEmit_ARMISD_VRSHRuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8325 if (RetVT.SimpleTy != MVT::v4i32)
8326 return Register();
8327 if ((Subtarget->hasNEON())) {
8328 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8329 }
8330 return Register();
8331}
8332
8333Register fastEmit_ARMISD_VRSHRuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8334 if (RetVT.SimpleTy != MVT::v1i64)
8335 return Register();
8336 if ((Subtarget->hasNEON())) {
8337 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8338 }
8339 return Register();
8340}
8341
8342Register fastEmit_ARMISD_VRSHRuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8343 if (RetVT.SimpleTy != MVT::v2i64)
8344 return Register();
8345 if ((Subtarget->hasNEON())) {
8346 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRuv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8347 }
8348 return Register();
8349}
8350
8351Register fastEmit_ARMISD_VRSHRuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8352 switch (VT.SimpleTy) {
8353 case MVT::v8i8: return fastEmit_ARMISD_VRSHRuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8354 case MVT::v16i8: return fastEmit_ARMISD_VRSHRuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8355 case MVT::v4i16: return fastEmit_ARMISD_VRSHRuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8356 case MVT::v8i16: return fastEmit_ARMISD_VRSHRuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8357 case MVT::v2i32: return fastEmit_ARMISD_VRSHRuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8358 case MVT::v4i32: return fastEmit_ARMISD_VRSHRuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8359 case MVT::v1i64: return fastEmit_ARMISD_VRSHRuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8360 case MVT::v2i64: return fastEmit_ARMISD_VRSHRuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8361 default: return Register();
8362 }
8363}
8364
8365// FastEmit functions for ARMISD::VSHLIMM.
8366
8367Register fastEmit_ARMISD_VSHLIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8368 if (RetVT.SimpleTy != MVT::v8i8)
8369 return Register();
8370 if ((Subtarget->hasNEON())) {
8371 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8372 }
8373 return Register();
8374}
8375
8376Register fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8377 if (RetVT.SimpleTy != MVT::v16i8)
8378 return Register();
8379 if ((Subtarget->hasNEON())) {
8380 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8381 }
8382 return Register();
8383}
8384
8385Register fastEmit_ARMISD_VSHLIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8386 if (RetVT.SimpleTy != MVT::v4i16)
8387 return Register();
8388 if ((Subtarget->hasNEON())) {
8389 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8390 }
8391 return Register();
8392}
8393
8394Register fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8395 if (RetVT.SimpleTy != MVT::v8i16)
8396 return Register();
8397 if ((Subtarget->hasNEON())) {
8398 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8399 }
8400 return Register();
8401}
8402
8403Register fastEmit_ARMISD_VSHLIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8404 if (RetVT.SimpleTy != MVT::v2i32)
8405 return Register();
8406 if ((Subtarget->hasNEON())) {
8407 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8408 }
8409 return Register();
8410}
8411
8412Register fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8413 if (RetVT.SimpleTy != MVT::v4i32)
8414 return Register();
8415 if ((Subtarget->hasNEON())) {
8416 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8417 }
8418 return Register();
8419}
8420
8421Register fastEmit_ARMISD_VSHLIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8422 if (RetVT.SimpleTy != MVT::v1i64)
8423 return Register();
8424 if ((Subtarget->hasNEON())) {
8425 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8426 }
8427 return Register();
8428}
8429
8430Register fastEmit_ARMISD_VSHLIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8431 if (RetVT.SimpleTy != MVT::v2i64)
8432 return Register();
8433 if ((Subtarget->hasNEON())) {
8434 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHLiv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8435 }
8436 return Register();
8437}
8438
8439Register fastEmit_ARMISD_VSHLIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8440 switch (VT.SimpleTy) {
8441 case MVT::v8i8: return fastEmit_ARMISD_VSHLIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8442 case MVT::v16i8: return fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8443 case MVT::v4i16: return fastEmit_ARMISD_VSHLIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8444 case MVT::v8i16: return fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8445 case MVT::v2i32: return fastEmit_ARMISD_VSHLIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8446 case MVT::v4i32: return fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8447 case MVT::v1i64: return fastEmit_ARMISD_VSHLIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8448 case MVT::v2i64: return fastEmit_ARMISD_VSHLIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8449 default: return Register();
8450 }
8451}
8452
8453// FastEmit functions for ARMISD::VSHRsIMM.
8454
8455Register fastEmit_ARMISD_VSHRsIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8456 if (RetVT.SimpleTy != MVT::v8i8)
8457 return Register();
8458 if ((Subtarget->hasNEON())) {
8459 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8460 }
8461 return Register();
8462}
8463
8464Register fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8465 if (RetVT.SimpleTy != MVT::v16i8)
8466 return Register();
8467 if ((Subtarget->hasNEON())) {
8468 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8469 }
8470 return Register();
8471}
8472
8473Register fastEmit_ARMISD_VSHRsIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8474 if (RetVT.SimpleTy != MVT::v4i16)
8475 return Register();
8476 if ((Subtarget->hasNEON())) {
8477 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8478 }
8479 return Register();
8480}
8481
8482Register fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8483 if (RetVT.SimpleTy != MVT::v8i16)
8484 return Register();
8485 if ((Subtarget->hasNEON())) {
8486 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8487 }
8488 return Register();
8489}
8490
8491Register fastEmit_ARMISD_VSHRsIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8492 if (RetVT.SimpleTy != MVT::v2i32)
8493 return Register();
8494 if ((Subtarget->hasNEON())) {
8495 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8496 }
8497 return Register();
8498}
8499
8500Register fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8501 if (RetVT.SimpleTy != MVT::v4i32)
8502 return Register();
8503 if ((Subtarget->hasNEON())) {
8504 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8505 }
8506 return Register();
8507}
8508
8509Register fastEmit_ARMISD_VSHRsIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8510 if (RetVT.SimpleTy != MVT::v1i64)
8511 return Register();
8512 if ((Subtarget->hasNEON())) {
8513 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8514 }
8515 return Register();
8516}
8517
8518Register fastEmit_ARMISD_VSHRsIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8519 if (RetVT.SimpleTy != MVT::v2i64)
8520 return Register();
8521 if ((Subtarget->hasNEON())) {
8522 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRsv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8523 }
8524 return Register();
8525}
8526
8527Register fastEmit_ARMISD_VSHRsIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8528 switch (VT.SimpleTy) {
8529 case MVT::v8i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8530 case MVT::v16i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8531 case MVT::v4i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8532 case MVT::v8i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8533 case MVT::v2i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8534 case MVT::v4i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8535 case MVT::v1i64: return fastEmit_ARMISD_VSHRsIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8536 case MVT::v2i64: return fastEmit_ARMISD_VSHRsIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8537 default: return Register();
8538 }
8539}
8540
8541// FastEmit functions for ARMISD::VSHRuIMM.
8542
8543Register fastEmit_ARMISD_VSHRuIMM_MVT_v8i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8544 if (RetVT.SimpleTy != MVT::v8i8)
8545 return Register();
8546 if ((Subtarget->hasNEON())) {
8547 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8548 }
8549 return Register();
8550}
8551
8552Register fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8553 if (RetVT.SimpleTy != MVT::v16i8)
8554 return Register();
8555 if ((Subtarget->hasNEON())) {
8556 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv16i8, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8557 }
8558 return Register();
8559}
8560
8561Register fastEmit_ARMISD_VSHRuIMM_MVT_v4i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8562 if (RetVT.SimpleTy != MVT::v4i16)
8563 return Register();
8564 if ((Subtarget->hasNEON())) {
8565 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8566 }
8567 return Register();
8568}
8569
8570Register fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8571 if (RetVT.SimpleTy != MVT::v8i16)
8572 return Register();
8573 if ((Subtarget->hasNEON())) {
8574 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv8i16, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8575 }
8576 return Register();
8577}
8578
8579Register fastEmit_ARMISD_VSHRuIMM_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8580 if (RetVT.SimpleTy != MVT::v2i32)
8581 return Register();
8582 if ((Subtarget->hasNEON())) {
8583 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8584 }
8585 return Register();
8586}
8587
8588Register fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8589 if (RetVT.SimpleTy != MVT::v4i32)
8590 return Register();
8591 if ((Subtarget->hasNEON())) {
8592 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv4i32, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8593 }
8594 return Register();
8595}
8596
8597Register fastEmit_ARMISD_VSHRuIMM_MVT_v1i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8598 if (RetVT.SimpleTy != MVT::v1i64)
8599 return Register();
8600 if ((Subtarget->hasNEON())) {
8601 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv1i64, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
8602 }
8603 return Register();
8604}
8605
8606Register fastEmit_ARMISD_VSHRuIMM_MVT_v2i64_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8607 if (RetVT.SimpleTy != MVT::v2i64)
8608 return Register();
8609 if ((Subtarget->hasNEON())) {
8610 return fastEmitInst_ri(MachineInstOpcode: ARM::VSHRuv2i64, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
8611 }
8612 return Register();
8613}
8614
8615Register fastEmit_ARMISD_VSHRuIMM_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8616 switch (VT.SimpleTy) {
8617 case MVT::v8i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i8_ri(RetVT, Op0, imm1);
8618 case MVT::v16i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri(RetVT, Op0, imm1);
8619 case MVT::v4i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i16_ri(RetVT, Op0, imm1);
8620 case MVT::v8i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri(RetVT, Op0, imm1);
8621 case MVT::v2i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v2i32_ri(RetVT, Op0, imm1);
8622 case MVT::v4i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri(RetVT, Op0, imm1);
8623 case MVT::v1i64: return fastEmit_ARMISD_VSHRuIMM_MVT_v1i64_ri(RetVT, Op0, imm1);
8624 case MVT::v2i64: return fastEmit_ARMISD_VSHRuIMM_MVT_v2i64_ri(RetVT, Op0, imm1);
8625 default: return Register();
8626 }
8627}
8628
8629// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
8630
8631Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8632 if (RetVT.SimpleTy != MVT::i32)
8633 return Register();
8634 if ((Subtarget->hasFPRegs()) && (!Subtarget->hasSlowVGETLNi32())) {
8635 return fastEmitInst_ri(MachineInstOpcode: ARM::VGETLNi32, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8636 }
8637 return Register();
8638}
8639
8640Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8641 switch (VT.SimpleTy) {
8642 case MVT::v2i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i32_ri(RetVT, Op0, imm1);
8643 default: return Register();
8644 }
8645}
8646
8647// FastEmit functions for ISD::SHL.
8648
8649Register fastEmit_ISD_SHL_MVT_i32_ri(MVT RetVT, Register Op0, uint64_t imm1) {
8650 if (RetVT.SimpleTy != MVT::i32)
8651 return Register();
8652 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
8653 return fastEmitInst_ri(MachineInstOpcode: ARM::tLSLri, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
8654 }
8655 return Register();
8656}
8657
8658Register fastEmit_ISD_SHL_ri(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8659 switch (VT.SimpleTy) {
8660 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri(RetVT, Op0, imm1);
8661 default: return Register();
8662 }
8663}
8664
8665// Top-level FastEmit function.
8666
8667Register fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) override {
8668 if (VT == MVT::i32 && Predicate_mod_imm(Imm: imm1))
8669 if (Register Reg = fastEmit_ri_Predicate_mod_imm(VT, RetVT, Opcode, Op0, imm1))
8670 return Reg;
8671
8672 if (VT == MVT::i32 && Predicate_imm0_7(Imm: imm1))
8673 if (Register Reg = fastEmit_ri_Predicate_imm0_7(VT, RetVT, Opcode, Op0, imm1))
8674 return Reg;
8675
8676 if (VT == MVT::i32 && Predicate_imm0_255_expr(Imm: imm1))
8677 if (Register Reg = fastEmit_ri_Predicate_imm0_255_expr(VT, RetVT, Opcode, Op0, imm1))
8678 return Reg;
8679
8680 if (VT == MVT::i32 && Predicate_imm0_255(Imm: imm1))
8681 if (Register Reg = fastEmit_ri_Predicate_imm0_255(VT, RetVT, Opcode, Op0, imm1))
8682 return Reg;
8683
8684 if (VT == MVT::i32 && Predicate_t2_so_imm(Imm: imm1))
8685 if (Register Reg = fastEmit_ri_Predicate_t2_so_imm(VT, RetVT, Opcode, Op0, imm1))
8686 return Reg;
8687
8688 if (VT == MVT::i32 && Predicate_imm0_4095(Imm: imm1))
8689 if (Register Reg = fastEmit_ri_Predicate_imm0_4095(VT, RetVT, Opcode, Op0, imm1))
8690 return Reg;
8691
8692 if (VT == MVT::i32 && Predicate_imm1_31(Imm: imm1))
8693 if (Register Reg = fastEmit_ri_Predicate_imm1_31(VT, RetVT, Opcode, Op0, imm1))
8694 return Reg;
8695
8696 if (VT == MVT::i32 && Predicate_shr_imm8(Imm: imm1))
8697 if (Register Reg = fastEmit_ri_Predicate_shr_imm8(VT, RetVT, Opcode, Op0, imm1))
8698 return Reg;
8699
8700 if (VT == MVT::i32 && Predicate_shr_imm16(Imm: imm1))
8701 if (Register Reg = fastEmit_ri_Predicate_shr_imm16(VT, RetVT, Opcode, Op0, imm1))
8702 return Reg;
8703
8704 if (VT == MVT::i32 && Predicate_shr_imm32(Imm: imm1))
8705 if (Register Reg = fastEmit_ri_Predicate_shr_imm32(VT, RetVT, Opcode, Op0, imm1))
8706 return Reg;
8707
8708 if (VT == MVT::i32 && Predicate_VectorIndex32(Imm: imm1))
8709 if (Register Reg = fastEmit_ri_Predicate_VectorIndex32(VT, RetVT, Opcode, Op0, imm1))
8710 return Reg;
8711
8712 if (VT == MVT::i32 && Predicate_imm0_31(Imm: imm1))
8713 if (Register Reg = fastEmit_ri_Predicate_imm0_31(VT, RetVT, Opcode, Op0, imm1))
8714 return Reg;
8715
8716 if (VT == MVT::i32 && Predicate_imm0_15(Imm: imm1))
8717 if (Register Reg = fastEmit_ri_Predicate_imm0_15(VT, RetVT, Opcode, Op0, imm1))
8718 return Reg;
8719
8720 switch (Opcode) {
8721 case ARMISD::PIC_ADD: return fastEmit_ARMISD_PIC_ADD_ri(VT, RetVT, Op0, imm1);
8722 case ARMISD::VDUPLANE: return fastEmit_ARMISD_VDUPLANE_ri(VT, RetVT, Op0, imm1);
8723 case ARMISD::VGETLANEs: return fastEmit_ARMISD_VGETLANEs_ri(VT, RetVT, Op0, imm1);
8724 case ARMISD::VGETLANEu: return fastEmit_ARMISD_VGETLANEu_ri(VT, RetVT, Op0, imm1);
8725 case ARMISD::VQSHLsIMM: return fastEmit_ARMISD_VQSHLsIMM_ri(VT, RetVT, Op0, imm1);
8726 case ARMISD::VQSHLsuIMM: return fastEmit_ARMISD_VQSHLsuIMM_ri(VT, RetVT, Op0, imm1);
8727 case ARMISD::VQSHLuIMM: return fastEmit_ARMISD_VQSHLuIMM_ri(VT, RetVT, Op0, imm1);
8728 case ARMISD::VRSHRsIMM: return fastEmit_ARMISD_VRSHRsIMM_ri(VT, RetVT, Op0, imm1);
8729 case ARMISD::VRSHRuIMM: return fastEmit_ARMISD_VRSHRuIMM_ri(VT, RetVT, Op0, imm1);
8730 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri(VT, RetVT, Op0, imm1);
8731 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri(VT, RetVT, Op0, imm1);
8732 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri(VT, RetVT, Op0, imm1);
8733 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(VT, RetVT, Op0, imm1);
8734 case ISD::SHL: return fastEmit_ISD_SHL_ri(VT, RetVT, Op0, imm1);
8735 default: return Register();
8736 }
8737}
8738
8739// FastEmit functions for ARMISD::CMN.
8740
8741Register fastEmit_ARMISD_CMN_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8742 if (RetVT.SimpleTy != MVT::i32)
8743 return Register();
8744 if ((!Subtarget->isThumb())) {
8745 return fastEmitInst_ri(MachineInstOpcode: ARM::CMNri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8746 }
8747 return Register();
8748}
8749
8750Register fastEmit_ARMISD_CMN_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8751 switch (VT.SimpleTy) {
8752 case MVT::i32: return fastEmit_ARMISD_CMN_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8753 default: return Register();
8754 }
8755}
8756
8757// FastEmit functions for ARMISD::CMP.
8758
8759Register fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8760 if (RetVT.SimpleTy != MVT::i32)
8761 return Register();
8762 if ((!Subtarget->isThumb())) {
8763 return fastEmitInst_ri(MachineInstOpcode: ARM::CMPri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8764 }
8765 return Register();
8766}
8767
8768Register fastEmit_ARMISD_CMP_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8769 switch (VT.SimpleTy) {
8770 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8771 default: return Register();
8772 }
8773}
8774
8775// FastEmit functions for ARMISD::CMPZ.
8776
8777Register fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8778 if (RetVT.SimpleTy != MVT::i32)
8779 return Register();
8780 if ((!Subtarget->isThumb())) {
8781 return fastEmitInst_ri(MachineInstOpcode: ARM::CMPri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8782 }
8783 return Register();
8784}
8785
8786Register fastEmit_ARMISD_CMPZ_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8787 switch (VT.SimpleTy) {
8788 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8789 default: return Register();
8790 }
8791}
8792
8793// FastEmit functions for ISD::ADD.
8794
8795Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8796 if (RetVT.SimpleTy != MVT::i32)
8797 return Register();
8798 if ((!Subtarget->isThumb())) {
8799 return fastEmitInst_ri(MachineInstOpcode: ARM::ADDri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8800 }
8801 return Register();
8802}
8803
8804Register fastEmit_ISD_ADD_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8805 switch (VT.SimpleTy) {
8806 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8807 default: return Register();
8808 }
8809}
8810
8811// FastEmit functions for ISD::AND.
8812
8813Register fastEmit_ISD_AND_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8814 if (RetVT.SimpleTy != MVT::i32)
8815 return Register();
8816 if ((!Subtarget->isThumb())) {
8817 return fastEmitInst_ri(MachineInstOpcode: ARM::ANDri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8818 }
8819 return Register();
8820}
8821
8822Register fastEmit_ISD_AND_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8823 switch (VT.SimpleTy) {
8824 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8825 default: return Register();
8826 }
8827}
8828
8829// FastEmit functions for ISD::OR.
8830
8831Register fastEmit_ISD_OR_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8832 if (RetVT.SimpleTy != MVT::i32)
8833 return Register();
8834 if ((!Subtarget->isThumb())) {
8835 return fastEmitInst_ri(MachineInstOpcode: ARM::ORRri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8836 }
8837 return Register();
8838}
8839
8840Register fastEmit_ISD_OR_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8841 switch (VT.SimpleTy) {
8842 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8843 default: return Register();
8844 }
8845}
8846
8847// FastEmit functions for ISD::SUB.
8848
8849Register fastEmit_ISD_SUB_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8850 if (RetVT.SimpleTy != MVT::i32)
8851 return Register();
8852 if ((!Subtarget->isThumb())) {
8853 return fastEmitInst_ri(MachineInstOpcode: ARM::SUBri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8854 }
8855 return Register();
8856}
8857
8858Register fastEmit_ISD_SUB_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8859 switch (VT.SimpleTy) {
8860 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8861 default: return Register();
8862 }
8863}
8864
8865// FastEmit functions for ISD::XOR.
8866
8867Register fastEmit_ISD_XOR_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, Register Op0, uint64_t imm1) {
8868 if (RetVT.SimpleTy != MVT::i32)
8869 return Register();
8870 if ((!Subtarget->isThumb())) {
8871 return fastEmitInst_ri(MachineInstOpcode: ARM::EORri, RC: &ARM::GPRRegClass, Op0, Imm: imm1);
8872 }
8873 return Register();
8874}
8875
8876Register fastEmit_ISD_XOR_ri_Predicate_mod_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8877 switch (VT.SimpleTy) {
8878 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, imm1);
8879 default: return Register();
8880 }
8881}
8882
8883// Top-level FastEmit function.
8884
8885Register fastEmit_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8886 switch (Opcode) {
8887 case ARMISD::CMN: return fastEmit_ARMISD_CMN_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8888 case ARMISD::CMP: return fastEmit_ARMISD_CMP_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8889 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8890 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8891 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8892 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8893 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8894 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_mod_imm(VT, RetVT, Op0, imm1);
8895 default: return Register();
8896 }
8897}
8898
8899// FastEmit functions for ARMISD::VSHLIMM.
8900
8901Register fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
8902 if (RetVT.SimpleTy != MVT::v16i8)
8903 return Register();
8904 if ((Subtarget->hasMVEIntegerOps())) {
8905 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHL_immi8, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8906 }
8907 return Register();
8908}
8909
8910Register fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8911 switch (VT.SimpleTy) {
8912 case MVT::v16i8: return fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, imm1);
8913 default: return Register();
8914 }
8915}
8916
8917// FastEmit functions for ARMISD::VSHRsIMM.
8918
8919Register fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
8920 if (RetVT.SimpleTy != MVT::v16i8)
8921 return Register();
8922 if ((Subtarget->hasMVEIntegerOps())) {
8923 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_imms8, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8924 }
8925 return Register();
8926}
8927
8928Register fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8929 switch (VT.SimpleTy) {
8930 case MVT::v16i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, imm1);
8931 default: return Register();
8932 }
8933}
8934
8935// FastEmit functions for ARMISD::VSHRuIMM.
8936
8937Register fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
8938 if (RetVT.SimpleTy != MVT::v16i8)
8939 return Register();
8940 if ((Subtarget->hasMVEIntegerOps())) {
8941 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_immu8, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
8942 }
8943 return Register();
8944}
8945
8946Register fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8947 switch (VT.SimpleTy) {
8948 case MVT::v16i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, imm1);
8949 default: return Register();
8950 }
8951}
8952
8953// FastEmit functions for ISD::ADD.
8954
8955Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_7(MVT RetVT, Register Op0, uint64_t imm1) {
8956 if (RetVT.SimpleTy != MVT::i32)
8957 return Register();
8958 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
8959 return fastEmitInst_ri(MachineInstOpcode: ARM::tADDi3, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
8960 }
8961 return Register();
8962}
8963
8964Register fastEmit_ISD_ADD_ri_Predicate_imm0_7(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8965 switch (VT.SimpleTy) {
8966 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_7(RetVT, Op0, imm1);
8967 default: return Register();
8968 }
8969}
8970
8971// Top-level FastEmit function.
8972
8973Register fastEmit_ri_Predicate_imm0_7(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
8974 switch (Opcode) {
8975 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
8976 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
8977 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
8978 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm0_7(VT, RetVT, Op0, imm1);
8979 default: return Register();
8980 }
8981}
8982
8983// FastEmit functions for ISD::ADD.
8984
8985Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_255_expr(MVT RetVT, Register Op0, uint64_t imm1) {
8986 if (RetVT.SimpleTy != MVT::i32)
8987 return Register();
8988 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
8989 return fastEmitInst_ri(MachineInstOpcode: ARM::tADDi8, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
8990 }
8991 return Register();
8992}
8993
8994Register fastEmit_ISD_ADD_ri_Predicate_imm0_255_expr(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
8995 switch (VT.SimpleTy) {
8996 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_255_expr(RetVT, Op0, imm1);
8997 default: return Register();
8998 }
8999}
9000
9001// Top-level FastEmit function.
9002
9003Register fastEmit_ri_Predicate_imm0_255_expr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9004 switch (Opcode) {
9005 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm0_255_expr(VT, RetVT, Op0, imm1);
9006 default: return Register();
9007 }
9008}
9009
9010// FastEmit functions for ARMISD::CMP.
9011
9012Register fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_imm0_255(MVT RetVT, Register Op0, uint64_t imm1) {
9013 if (RetVT.SimpleTy != MVT::i32)
9014 return Register();
9015 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
9016 return fastEmitInst_ri(MachineInstOpcode: ARM::tCMPi8, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
9017 }
9018 return Register();
9019}
9020
9021Register fastEmit_ARMISD_CMP_ri_Predicate_imm0_255(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9022 switch (VT.SimpleTy) {
9023 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_imm0_255(RetVT, Op0, imm1);
9024 default: return Register();
9025 }
9026}
9027
9028// FastEmit functions for ARMISD::CMPZ.
9029
9030Register fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_imm0_255(MVT RetVT, Register Op0, uint64_t imm1) {
9031 if (RetVT.SimpleTy != MVT::i32)
9032 return Register();
9033 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
9034 return fastEmitInst_ri(MachineInstOpcode: ARM::tCMPi8, RC: &ARM::tGPRRegClass, Op0, Imm: imm1);
9035 }
9036 return Register();
9037}
9038
9039Register fastEmit_ARMISD_CMPZ_ri_Predicate_imm0_255(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9040 switch (VT.SimpleTy) {
9041 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_imm0_255(RetVT, Op0, imm1);
9042 default: return Register();
9043 }
9044}
9045
9046// Top-level FastEmit function.
9047
9048Register fastEmit_ri_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9049 switch (Opcode) {
9050 case ARMISD::CMP: return fastEmit_ARMISD_CMP_ri_Predicate_imm0_255(VT, RetVT, Op0, imm1);
9051 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_ri_Predicate_imm0_255(VT, RetVT, Op0, imm1);
9052 default: return Register();
9053 }
9054}
9055
9056// FastEmit functions for ARMISD::CMP.
9057
9058Register fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
9059 if (RetVT.SimpleTy != MVT::i32)
9060 return Register();
9061 if ((Subtarget->isThumb2())) {
9062 return fastEmitInst_ri(MachineInstOpcode: ARM::t2CMPri, RC: &ARM::GPRnopcRegClass, Op0, Imm: imm1);
9063 }
9064 return Register();
9065}
9066
9067Register fastEmit_ARMISD_CMP_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9068 switch (VT.SimpleTy) {
9069 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
9070 default: return Register();
9071 }
9072}
9073
9074// FastEmit functions for ARMISD::CMPZ.
9075
9076Register fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
9077 if (RetVT.SimpleTy != MVT::i32)
9078 return Register();
9079 if ((Subtarget->isThumb2())) {
9080 return fastEmitInst_ri(MachineInstOpcode: ARM::t2CMPri, RC: &ARM::GPRnopcRegClass, Op0, Imm: imm1);
9081 }
9082 return Register();
9083}
9084
9085Register fastEmit_ARMISD_CMPZ_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9086 switch (VT.SimpleTy) {
9087 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
9088 default: return Register();
9089 }
9090}
9091
9092// FastEmit functions for ISD::ADD.
9093
9094Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
9095 if (RetVT.SimpleTy != MVT::i32)
9096 return Register();
9097 if ((Subtarget->isThumb2())) {
9098 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ADDri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
9099 }
9100 return Register();
9101}
9102
9103Register fastEmit_ISD_ADD_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9104 switch (VT.SimpleTy) {
9105 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
9106 default: return Register();
9107 }
9108}
9109
9110// FastEmit functions for ISD::AND.
9111
9112Register fastEmit_ISD_AND_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
9113 if (RetVT.SimpleTy != MVT::i32)
9114 return Register();
9115 if ((Subtarget->isThumb2())) {
9116 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ANDri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
9117 }
9118 return Register();
9119}
9120
9121Register fastEmit_ISD_AND_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9122 switch (VT.SimpleTy) {
9123 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
9124 default: return Register();
9125 }
9126}
9127
9128// FastEmit functions for ISD::OR.
9129
9130Register fastEmit_ISD_OR_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
9131 if (RetVT.SimpleTy != MVT::i32)
9132 return Register();
9133 if ((Subtarget->isThumb2())) {
9134 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ORRri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
9135 }
9136 return Register();
9137}
9138
9139Register fastEmit_ISD_OR_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9140 switch (VT.SimpleTy) {
9141 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
9142 default: return Register();
9143 }
9144}
9145
9146// FastEmit functions for ISD::SUB.
9147
9148Register fastEmit_ISD_SUB_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
9149 if (RetVT.SimpleTy != MVT::i32)
9150 return Register();
9151 if ((Subtarget->isThumb2())) {
9152 return fastEmitInst_ri(MachineInstOpcode: ARM::t2SUBri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
9153 }
9154 return Register();
9155}
9156
9157Register fastEmit_ISD_SUB_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9158 switch (VT.SimpleTy) {
9159 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
9160 default: return Register();
9161 }
9162}
9163
9164// FastEmit functions for ISD::XOR.
9165
9166Register fastEmit_ISD_XOR_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, Register Op0, uint64_t imm1) {
9167 if (RetVT.SimpleTy != MVT::i32)
9168 return Register();
9169 if ((Subtarget->isThumb2())) {
9170 return fastEmitInst_ri(MachineInstOpcode: ARM::t2EORri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
9171 }
9172 return Register();
9173}
9174
9175Register fastEmit_ISD_XOR_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9176 switch (VT.SimpleTy) {
9177 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, imm1);
9178 default: return Register();
9179 }
9180}
9181
9182// Top-level FastEmit function.
9183
9184Register fastEmit_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9185 switch (Opcode) {
9186 case ARMISD::CMP: return fastEmit_ARMISD_CMP_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
9187 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
9188 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
9189 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
9190 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
9191 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
9192 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_t2_so_imm(VT, RetVT, Op0, imm1);
9193 default: return Register();
9194 }
9195}
9196
9197// FastEmit functions for ISD::ADD.
9198
9199Register fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_4095(MVT RetVT, Register Op0, uint64_t imm1) {
9200 if (RetVT.SimpleTy != MVT::i32)
9201 return Register();
9202 if ((Subtarget->isThumb2())) {
9203 return fastEmitInst_ri(MachineInstOpcode: ARM::t2ADDri12, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
9204 }
9205 return Register();
9206}
9207
9208Register fastEmit_ISD_ADD_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9209 switch (VT.SimpleTy) {
9210 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_4095(RetVT, Op0, imm1);
9211 default: return Register();
9212 }
9213}
9214
9215// FastEmit functions for ISD::SUB.
9216
9217Register fastEmit_ISD_SUB_MVT_i32_ri_Predicate_imm0_4095(MVT RetVT, Register Op0, uint64_t imm1) {
9218 if (RetVT.SimpleTy != MVT::i32)
9219 return Register();
9220 if ((Subtarget->isThumb2())) {
9221 return fastEmitInst_ri(MachineInstOpcode: ARM::t2SUBri12, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
9222 }
9223 return Register();
9224}
9225
9226Register fastEmit_ISD_SUB_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9227 switch (VT.SimpleTy) {
9228 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_imm0_4095(RetVT, Op0, imm1);
9229 default: return Register();
9230 }
9231}
9232
9233// Top-level FastEmit function.
9234
9235Register fastEmit_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9236 switch (Opcode) {
9237 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm0_4095(VT, RetVT, Op0, imm1);
9238 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_imm0_4095(VT, RetVT, Op0, imm1);
9239 default: return Register();
9240 }
9241}
9242
9243// FastEmit functions for ISD::ROTR.
9244
9245Register fastEmit_ISD_ROTR_MVT_i32_ri_Predicate_imm1_31(MVT RetVT, Register Op0, uint64_t imm1) {
9246 if (RetVT.SimpleTy != MVT::i32)
9247 return Register();
9248 if ((Subtarget->isThumb2())) {
9249 return fastEmitInst_ri(MachineInstOpcode: ARM::t2RORri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
9250 }
9251 return Register();
9252}
9253
9254Register fastEmit_ISD_ROTR_ri_Predicate_imm1_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9255 switch (VT.SimpleTy) {
9256 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_ri_Predicate_imm1_31(RetVT, Op0, imm1);
9257 default: return Register();
9258 }
9259}
9260
9261// FastEmit functions for ISD::SHL.
9262
9263Register fastEmit_ISD_SHL_MVT_i32_ri_Predicate_imm1_31(MVT RetVT, Register Op0, uint64_t imm1) {
9264 if (RetVT.SimpleTy != MVT::i32)
9265 return Register();
9266 if ((Subtarget->isThumb2())) {
9267 return fastEmitInst_ri(MachineInstOpcode: ARM::t2LSLri, RC: &ARM::rGPRRegClass, Op0, Imm: imm1);
9268 }
9269 return Register();
9270}
9271
9272Register fastEmit_ISD_SHL_ri_Predicate_imm1_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9273 switch (VT.SimpleTy) {
9274 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri_Predicate_imm1_31(RetVT, Op0, imm1);
9275 default: return Register();
9276 }
9277}
9278
9279// Top-level FastEmit function.
9280
9281Register fastEmit_ri_Predicate_imm1_31(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9282 switch (Opcode) {
9283 case ISD::ROTR: return fastEmit_ISD_ROTR_ri_Predicate_imm1_31(VT, RetVT, Op0, imm1);
9284 case ISD::SHL: return fastEmit_ISD_SHL_ri_Predicate_imm1_31(VT, RetVT, Op0, imm1);
9285 default: return Register();
9286 }
9287}
9288
9289// FastEmit functions for ARMISD::VQRSHRNsIMM.
9290
9291Register fastEmit_ARMISD_VQRSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
9292 if (RetVT.SimpleTy != MVT::v8i8)
9293 return Register();
9294 if ((Subtarget->hasNEON())) {
9295 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9296 }
9297 return Register();
9298}
9299
9300Register fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9301 switch (VT.SimpleTy) {
9302 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
9303 default: return Register();
9304 }
9305}
9306
9307// FastEmit functions for ARMISD::VQRSHRNsuIMM.
9308
9309Register fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
9310 if (RetVT.SimpleTy != MVT::v8i8)
9311 return Register();
9312 if ((Subtarget->hasNEON())) {
9313 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRUNv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9314 }
9315 return Register();
9316}
9317
9318Register fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9319 switch (VT.SimpleTy) {
9320 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
9321 default: return Register();
9322 }
9323}
9324
9325// FastEmit functions for ARMISD::VQRSHRNuIMM.
9326
9327Register fastEmit_ARMISD_VQRSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
9328 if (RetVT.SimpleTy != MVT::v8i8)
9329 return Register();
9330 if ((Subtarget->hasNEON())) {
9331 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9332 }
9333 return Register();
9334}
9335
9336Register fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9337 switch (VT.SimpleTy) {
9338 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
9339 default: return Register();
9340 }
9341}
9342
9343// FastEmit functions for ARMISD::VQSHRNsIMM.
9344
9345Register fastEmit_ARMISD_VQSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
9346 if (RetVT.SimpleTy != MVT::v8i8)
9347 return Register();
9348 if ((Subtarget->hasNEON())) {
9349 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNsv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9350 }
9351 return Register();
9352}
9353
9354Register fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9355 switch (VT.SimpleTy) {
9356 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
9357 default: return Register();
9358 }
9359}
9360
9361// FastEmit functions for ARMISD::VQSHRNsuIMM.
9362
9363Register fastEmit_ARMISD_VQSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
9364 if (RetVT.SimpleTy != MVT::v8i8)
9365 return Register();
9366 if ((Subtarget->hasNEON())) {
9367 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRUNv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9368 }
9369 return Register();
9370}
9371
9372Register fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9373 switch (VT.SimpleTy) {
9374 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
9375 default: return Register();
9376 }
9377}
9378
9379// FastEmit functions for ARMISD::VQSHRNuIMM.
9380
9381Register fastEmit_ARMISD_VQSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
9382 if (RetVT.SimpleTy != MVT::v8i8)
9383 return Register();
9384 if ((Subtarget->hasNEON())) {
9385 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNuv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9386 }
9387 return Register();
9388}
9389
9390Register fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9391 switch (VT.SimpleTy) {
9392 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
9393 default: return Register();
9394 }
9395}
9396
9397// FastEmit functions for ARMISD::VRSHRNIMM.
9398
9399Register fastEmit_ARMISD_VRSHRNIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, Register Op0, uint64_t imm1) {
9400 if (RetVT.SimpleTy != MVT::v8i8)
9401 return Register();
9402 if ((Subtarget->hasNEON())) {
9403 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRNv8i8, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9404 }
9405 return Register();
9406}
9407
9408Register fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9409 switch (VT.SimpleTy) {
9410 case MVT::v8i16: return fastEmit_ARMISD_VRSHRNIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, imm1);
9411 default: return Register();
9412 }
9413}
9414
9415// Top-level FastEmit function.
9416
9417Register fastEmit_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9418 switch (Opcode) {
9419 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
9420 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
9421 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
9422 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
9423 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
9424 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
9425 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, imm1);
9426 default: return Register();
9427 }
9428}
9429
9430// FastEmit functions for ARMISD::VQRSHRNsIMM.
9431
9432Register fastEmit_ARMISD_VQRSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
9433 if (RetVT.SimpleTy != MVT::v4i16)
9434 return Register();
9435 if ((Subtarget->hasNEON())) {
9436 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9437 }
9438 return Register();
9439}
9440
9441Register fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9442 switch (VT.SimpleTy) {
9443 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
9444 default: return Register();
9445 }
9446}
9447
9448// FastEmit functions for ARMISD::VQRSHRNsuIMM.
9449
9450Register fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
9451 if (RetVT.SimpleTy != MVT::v4i16)
9452 return Register();
9453 if ((Subtarget->hasNEON())) {
9454 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRUNv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9455 }
9456 return Register();
9457}
9458
9459Register fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9460 switch (VT.SimpleTy) {
9461 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
9462 default: return Register();
9463 }
9464}
9465
9466// FastEmit functions for ARMISD::VQRSHRNuIMM.
9467
9468Register fastEmit_ARMISD_VQRSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
9469 if (RetVT.SimpleTy != MVT::v4i16)
9470 return Register();
9471 if ((Subtarget->hasNEON())) {
9472 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9473 }
9474 return Register();
9475}
9476
9477Register fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9478 switch (VT.SimpleTy) {
9479 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
9480 default: return Register();
9481 }
9482}
9483
9484// FastEmit functions for ARMISD::VQSHRNsIMM.
9485
9486Register fastEmit_ARMISD_VQSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
9487 if (RetVT.SimpleTy != MVT::v4i16)
9488 return Register();
9489 if ((Subtarget->hasNEON())) {
9490 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNsv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9491 }
9492 return Register();
9493}
9494
9495Register fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9496 switch (VT.SimpleTy) {
9497 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
9498 default: return Register();
9499 }
9500}
9501
9502// FastEmit functions for ARMISD::VQSHRNsuIMM.
9503
9504Register fastEmit_ARMISD_VQSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
9505 if (RetVT.SimpleTy != MVT::v4i16)
9506 return Register();
9507 if ((Subtarget->hasNEON())) {
9508 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRUNv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9509 }
9510 return Register();
9511}
9512
9513Register fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9514 switch (VT.SimpleTy) {
9515 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
9516 default: return Register();
9517 }
9518}
9519
9520// FastEmit functions for ARMISD::VQSHRNuIMM.
9521
9522Register fastEmit_ARMISD_VQSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
9523 if (RetVT.SimpleTy != MVT::v4i16)
9524 return Register();
9525 if ((Subtarget->hasNEON())) {
9526 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNuv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9527 }
9528 return Register();
9529}
9530
9531Register fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9532 switch (VT.SimpleTy) {
9533 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
9534 default: return Register();
9535 }
9536}
9537
9538// FastEmit functions for ARMISD::VRSHRNIMM.
9539
9540Register fastEmit_ARMISD_VRSHRNIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, Register Op0, uint64_t imm1) {
9541 if (RetVT.SimpleTy != MVT::v4i16)
9542 return Register();
9543 if ((Subtarget->hasNEON())) {
9544 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRNv4i16, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9545 }
9546 return Register();
9547}
9548
9549Register fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9550 switch (VT.SimpleTy) {
9551 case MVT::v4i32: return fastEmit_ARMISD_VRSHRNIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, imm1);
9552 default: return Register();
9553 }
9554}
9555
9556// Top-level FastEmit function.
9557
9558Register fastEmit_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9559 switch (Opcode) {
9560 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9561 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9562 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9563 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9564 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9565 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9566 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, imm1);
9567 default: return Register();
9568 }
9569}
9570
9571// FastEmit functions for ARMISD::VQRSHRNsIMM.
9572
9573Register fastEmit_ARMISD_VQRSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9574 if (RetVT.SimpleTy != MVT::v2i32)
9575 return Register();
9576 if ((Subtarget->hasNEON())) {
9577 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9578 }
9579 return Register();
9580}
9581
9582Register fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9583 switch (VT.SimpleTy) {
9584 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9585 default: return Register();
9586 }
9587}
9588
9589// FastEmit functions for ARMISD::VQRSHRNsuIMM.
9590
9591Register fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9592 if (RetVT.SimpleTy != MVT::v2i32)
9593 return Register();
9594 if ((Subtarget->hasNEON())) {
9595 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRUNv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9596 }
9597 return Register();
9598}
9599
9600Register fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9601 switch (VT.SimpleTy) {
9602 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9603 default: return Register();
9604 }
9605}
9606
9607// FastEmit functions for ARMISD::VQRSHRNuIMM.
9608
9609Register fastEmit_ARMISD_VQRSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9610 if (RetVT.SimpleTy != MVT::v2i32)
9611 return Register();
9612 if ((Subtarget->hasNEON())) {
9613 return fastEmitInst_ri(MachineInstOpcode: ARM::VQRSHRNuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9614 }
9615 return Register();
9616}
9617
9618Register fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9619 switch (VT.SimpleTy) {
9620 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9621 default: return Register();
9622 }
9623}
9624
9625// FastEmit functions for ARMISD::VQSHRNsIMM.
9626
9627Register fastEmit_ARMISD_VQSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9628 if (RetVT.SimpleTy != MVT::v2i32)
9629 return Register();
9630 if ((Subtarget->hasNEON())) {
9631 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNsv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9632 }
9633 return Register();
9634}
9635
9636Register fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9637 switch (VT.SimpleTy) {
9638 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9639 default: return Register();
9640 }
9641}
9642
9643// FastEmit functions for ARMISD::VQSHRNsuIMM.
9644
9645Register fastEmit_ARMISD_VQSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9646 if (RetVT.SimpleTy != MVT::v2i32)
9647 return Register();
9648 if ((Subtarget->hasNEON())) {
9649 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRUNv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9650 }
9651 return Register();
9652}
9653
9654Register fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9655 switch (VT.SimpleTy) {
9656 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9657 default: return Register();
9658 }
9659}
9660
9661// FastEmit functions for ARMISD::VQSHRNuIMM.
9662
9663Register fastEmit_ARMISD_VQSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9664 if (RetVT.SimpleTy != MVT::v2i32)
9665 return Register();
9666 if ((Subtarget->hasNEON())) {
9667 return fastEmitInst_ri(MachineInstOpcode: ARM::VQSHRNuv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9668 }
9669 return Register();
9670}
9671
9672Register fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9673 switch (VT.SimpleTy) {
9674 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9675 default: return Register();
9676 }
9677}
9678
9679// FastEmit functions for ARMISD::VRSHRNIMM.
9680
9681Register fastEmit_ARMISD_VRSHRNIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, Register Op0, uint64_t imm1) {
9682 if (RetVT.SimpleTy != MVT::v2i32)
9683 return Register();
9684 if ((Subtarget->hasNEON())) {
9685 return fastEmitInst_ri(MachineInstOpcode: ARM::VRSHRNv2i32, RC: &ARM::DPRRegClass, Op0, Imm: imm1);
9686 }
9687 return Register();
9688}
9689
9690Register fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9691 switch (VT.SimpleTy) {
9692 case MVT::v2i64: return fastEmit_ARMISD_VRSHRNIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, imm1);
9693 default: return Register();
9694 }
9695}
9696
9697// Top-level FastEmit function.
9698
9699Register fastEmit_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9700 switch (Opcode) {
9701 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9702 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9703 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9704 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9705 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9706 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9707 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, imm1);
9708 default: return Register();
9709 }
9710}
9711
9712// FastEmit functions for ARMISD::VDUPLANE.
9713
9714Register fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri_Predicate_VectorIndex32(MVT RetVT, Register Op0, uint64_t imm1) {
9715 if (RetVT.SimpleTy != MVT::v16i8)
9716 return Register();
9717 if ((Subtarget->hasNEON())) {
9718 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN8q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
9719 }
9720 return Register();
9721}
9722
9723Register fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri_Predicate_VectorIndex32(MVT RetVT, Register Op0, uint64_t imm1) {
9724 if (RetVT.SimpleTy != MVT::v8i16)
9725 return Register();
9726 if ((Subtarget->hasNEON())) {
9727 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN16q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
9728 }
9729 return Register();
9730}
9731
9732Register fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri_Predicate_VectorIndex32(MVT RetVT, Register Op0, uint64_t imm1) {
9733 if (RetVT.SimpleTy != MVT::v4i32)
9734 return Register();
9735 if ((Subtarget->hasNEON())) {
9736 return fastEmitInst_ri(MachineInstOpcode: ARM::VDUPLN32q, RC: &ARM::QPRRegClass, Op0, Imm: imm1);
9737 }
9738 return Register();
9739}
9740
9741Register fastEmit_ARMISD_VDUPLANE_ri_Predicate_VectorIndex32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9742 switch (VT.SimpleTy) {
9743 case MVT::v8i8: return fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri_Predicate_VectorIndex32(RetVT, Op0, imm1);
9744 case MVT::v4i16: return fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri_Predicate_VectorIndex32(RetVT, Op0, imm1);
9745 case MVT::v2i32: return fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri_Predicate_VectorIndex32(RetVT, Op0, imm1);
9746 default: return Register();
9747 }
9748}
9749
9750// Top-level FastEmit function.
9751
9752Register fastEmit_ri_Predicate_VectorIndex32(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9753 switch (Opcode) {
9754 case ARMISD::VDUPLANE: return fastEmit_ARMISD_VDUPLANE_ri_Predicate_VectorIndex32(VT, RetVT, Op0, imm1);
9755 default: return Register();
9756 }
9757}
9758
9759// FastEmit functions for ARMISD::VSHLIMM.
9760
9761Register fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, Register Op0, uint64_t imm1) {
9762 if (RetVT.SimpleTy != MVT::v4i32)
9763 return Register();
9764 if ((Subtarget->hasMVEIntegerOps())) {
9765 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHL_immi32, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9766 }
9767 return Register();
9768}
9769
9770Register fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9771 switch (VT.SimpleTy) {
9772 case MVT::v4i32: return fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, imm1);
9773 default: return Register();
9774 }
9775}
9776
9777// FastEmit functions for ARMISD::VSHRsIMM.
9778
9779Register fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, Register Op0, uint64_t imm1) {
9780 if (RetVT.SimpleTy != MVT::v4i32)
9781 return Register();
9782 if ((Subtarget->hasMVEIntegerOps())) {
9783 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_imms32, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9784 }
9785 return Register();
9786}
9787
9788Register fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9789 switch (VT.SimpleTy) {
9790 case MVT::v4i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, imm1);
9791 default: return Register();
9792 }
9793}
9794
9795// FastEmit functions for ARMISD::VSHRuIMM.
9796
9797Register fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, Register Op0, uint64_t imm1) {
9798 if (RetVT.SimpleTy != MVT::v4i32)
9799 return Register();
9800 if ((Subtarget->hasMVEIntegerOps())) {
9801 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_immu32, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9802 }
9803 return Register();
9804}
9805
9806Register fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9807 switch (VT.SimpleTy) {
9808 case MVT::v4i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, imm1);
9809 default: return Register();
9810 }
9811}
9812
9813// Top-level FastEmit function.
9814
9815Register fastEmit_ri_Predicate_imm0_31(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9816 switch (Opcode) {
9817 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, imm1);
9818 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, imm1);
9819 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, imm1);
9820 default: return Register();
9821 }
9822}
9823
9824// FastEmit functions for ARMISD::VSHLIMM.
9825
9826Register fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, Register Op0, uint64_t imm1) {
9827 if (RetVT.SimpleTy != MVT::v8i16)
9828 return Register();
9829 if ((Subtarget->hasMVEIntegerOps())) {
9830 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHL_immi16, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9831 }
9832 return Register();
9833}
9834
9835Register fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9836 switch (VT.SimpleTy) {
9837 case MVT::v8i16: return fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, imm1);
9838 default: return Register();
9839 }
9840}
9841
9842// FastEmit functions for ARMISD::VSHRsIMM.
9843
9844Register fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, Register Op0, uint64_t imm1) {
9845 if (RetVT.SimpleTy != MVT::v8i16)
9846 return Register();
9847 if ((Subtarget->hasMVEIntegerOps())) {
9848 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_imms16, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9849 }
9850 return Register();
9851}
9852
9853Register fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9854 switch (VT.SimpleTy) {
9855 case MVT::v8i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, imm1);
9856 default: return Register();
9857 }
9858}
9859
9860// FastEmit functions for ARMISD::VSHRuIMM.
9861
9862Register fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, Register Op0, uint64_t imm1) {
9863 if (RetVT.SimpleTy != MVT::v8i16)
9864 return Register();
9865 if ((Subtarget->hasMVEIntegerOps())) {
9866 return fastEmitInst_ri(MachineInstOpcode: ARM::MVE_VSHR_immu16, RC: &ARM::MQPRRegClass, Op0, Imm: imm1);
9867 }
9868 return Register();
9869}
9870
9871Register fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
9872 switch (VT.SimpleTy) {
9873 case MVT::v8i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, imm1);
9874 default: return Register();
9875 }
9876}
9877
9878// Top-level FastEmit function.
9879
9880Register fastEmit_ri_Predicate_imm0_15(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
9881 switch (Opcode) {
9882 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, imm1);
9883 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, imm1);
9884 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, imm1);
9885 default: return Register();
9886 }
9887}
9888
9889// FastEmit functions for ISD::Constant.
9890
9891Register fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
9892 if (RetVT.SimpleTy != MVT::i32)
9893 return Register();
9894 if ((Subtarget->isThumb()) && (Subtarget->useMovt())) {
9895 return fastEmitInst_i(MachineInstOpcode: ARM::t2MOVi32imm, RC: &ARM::rGPRRegClass, Imm: imm0);
9896 }
9897 if ((!Subtarget->useMovt()) && (Subtarget->genExecuteOnly()) && (Subtarget->isThumb1Only())) {
9898 return fastEmitInst_i(MachineInstOpcode: ARM::tMOVi32imm, RC: &ARM::rGPRRegClass, Imm: imm0);
9899 }
9900 return Register();
9901}
9902
9903Register fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
9904 switch (VT.SimpleTy) {
9905 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
9906 default: return Register();
9907 }
9908}
9909
9910// Top-level FastEmit function.
9911
9912Register fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
9913 switch (Opcode) {
9914 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
9915 default: return Register();
9916 }
9917}
9918
9919